unit test for CSV import fixes related to fdo#48621

3 files changed, 210 insertions, 1 deletions

diff --git a/sc/qa/unit/data/contentCSV/fdo48621_broken_quotes_exported.csv b/sc/qa/unit/data/contentCSV/fdo48621_broken_quotes_exported.csv
new file mode 100644
index 000000000000..b1c95af5c689
--- /dev/null
+++ b/sc/qa/unit/data/contentCSV/fdo48621_broken_quotes_exported.csv
@@ -0,0 +1,90 @@
+i80358,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+"abc"" d ""ef",shall be:,"abc"" d ""ef",,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+"a""b, ""a",shall be:,"a""b, ""a",,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+i78926_input.csv,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+2,5,GR010 015 1001,/image/grimas/015/1001.jpg,/image/grimas/015/1001.jpg,/image/grimas/015/1001.jpg,,,,,,,,,,,,,5.45,0,0000-00-00 00:00:00,0000-00-00 00:00:00,0000-00-00 00:00:00,0.05,1,2,81,4,0,1,0,2000,0,0,0,"0
+"
+3,5,GR010 015 1002,/image/grimas/015/1002.jpg,/image/grimas/015/1002.jpg,/image/grimas/015/1002.jpg,,,,,,,,,,,,,5.45,0,0000-00-00 00:00:00,0000-00-00 00:00:00,0000-00-00 00:00:00,0.05,1,2,81,4,0,0,0,2000,0,0,0,"0
+"
+,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+i78926_input.csv,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+772,3,SPAZZOLA ARIA BABYLISS 32,image/APP.ELET/Varie/Babyliss-Spazzola-Elettrica.jpg,image/APP.ELET/Varie/Babyliss-Spazzola-Elettrica.jpg,image/APP.ELET/Varie/Babyliss-Spazzola-Elettrica.jpg,,,,,,,,,,,,,33,0,2006-11-07 09:26:16,2007-05-24 17:54:24,NULL,1,1,2,86,150,0,0,0,2000,0,,,
+773,1,"BIGODINI RISCALDANTI \""BA",image/APP.ELET/Varie/Bigodini-caldi.jpg,image/APP.ELET/Varie/Bigodini-caldi.jpg,image/APP.ELET/Varie/Bigodini-caldi.jpg,,,,,,,,,,,,,57.91,0,2006-11-07 09:26:16,2006-11-07 09:26:16,0000-00-00 00:00:00,0.7,1,2,86,152,0,1,0,2000,0,0,0,0
+774,3,"PIASTRA \""BABYLISS\"" PRO C",image/APP.ELET/piastre/Piastre-Babyliss-2564E.jpg,image/APP.ELET/piastre/Piastre-Babyliss-2564E.jpg,image/APP.ELET/piastre/Piastre-Babyliss-2564E.jpg,,,,,,,,,,,,,78,0,2006-11-07 09:26:16,2006-12-13 19:18:31,0000-00-00 00:00:00,0.7,1,2,86,148,0,0,0,2000,0,0,0,0
+775,0,PHON ULTRON MAXI 2200W+1,image/APP.ELET/phon/Phon-Ultron-1500W.jpg,image/APP.ELET/phon/Phon-Ultron-1500W.jpg,image/APP.ELET/phon/Phon-Ultron-1500W.jpg,,,,,,,,,,,,,26.66,0,2006-11-07 09:26:16,2006-12-13 18:57:03,0000-00-00 00:00:00,0.7,1,2,101,158,0,0,0,2000,0,0,0,0
+,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+i78926_inconsistent_double_quote_behavior.csv,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+Description,Description,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+The following text breaks everything,"OpenOffice.org calc doesnt "" like quotes embedded in fields.",,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+Here is some data,Here is some data,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+Here is some data,Here is some data,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+Here is some data,Here is some data,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+Here is some data,Here is some data,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+This field will fix everything,"Here is the "" fix",,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+First fixed record,Here is some data,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+Here is some data,Here is some data,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+This field will break things again,"Mismatched quote "" must be after some quoted text",,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+Here is some data,Here is some data,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+Here is some data,Here is some data,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+Here is some data,Here is some data,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+This field will fix everything,"Here is the "" fix",,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+First fixed record,Here is some data,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+Here is some data,Here is some data,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+Here is some data,Here is some data,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+Here is some data,Here is some data,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+Here is some data,Here is some data,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+This next field will /not/ break things because /this/ field is not quoted,"Mismatched quote "" must be after some quoted text",,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+Here is some data,Here is some data,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+Here is some data,Here is some data,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+Here is some data,Here is some data,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+i80385_test1.csv,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+test,"a""b, ""a"," d""a""c ", m ,j ,d,"b""A""","D""E","f,1","a,b","de""b,a",,,,,,,,,,,,,,,,,,,,,,,,,
+"abcdef"" test ""abc","def""g""h","def""gh""",,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+"this is
+a test","yes
+it
+works",,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+No it doesn't,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+"yes it,"",""does",no,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+"""""a,cd""e
+""a""b""
+",<- needed to end test file here,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+i80385_test2.csv,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+test,"a""b, ""a"," d""a""c ", m ,j ,d,"b""A""","D""E","f,1","a,b","de""b,a
+""abcdef"" test ""abc","def""g""h","def""gh""",,,,,,,,,,,,,,,,,,,,,,,
+"this is
+a test","yes
+it
+works",,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+No it doesn't,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+"yes it,"",""does",no,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+"""""a,cd""e
+""a""b""
+",<- needed to end test file here,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+i80385_test3.csv,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+test,"a""b, ""a"," d""a""c ", m ,j ,d,"b""A""","D""E","f,1","a,b","de""b,a",,,,,,,,,,,,,,,,,,,,,,,,,
+"abcdef"" test ""abc","def""g""h","def""gh""",,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+"this is
+a test","yes
+it
+works",,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+No it doesn't,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+"yes it,"",""does",no,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+"""""a,cd""e
+""a""b""",,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+i80385_test4.csv,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+test,"a""b, ""a"," d""a""c ", m ,j ,d,"b""A""","D""E","f,1","a,b","de""b,a
+""abcdef"" test ""abc","def""g""h","def""gh""",,,,,,,,,,,,,,,,,,,,,,,
+"this is
+a test","yes
+it
+works",,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+No it doesn't,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+"yes it,"",""does",no,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+"""""a,cd""e
+""a""b""",,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
diff --git a/sc/qa/unit/data/csv/fdo48621_broken_quotes.csv b/sc/qa/unit/data/csv/fdo48621_broken_quotes.csv
new file mode 100644
index 000000000000..dbefc2481f84
--- /dev/null
+++ b/sc/qa/unit/data/csv/fdo48621_broken_quotes.csv
@@ -0,0 +1,88 @@
+i80358
+"abc" d "ef",shall be:,"abc"" d ""ef"
+"a"b, "a",shall be:,"a""b, ""a"
+
+i78926_input.csv
+"2","5","GR010 015 1001","/image/grimas/015/1001.jpg","/image/grimas/015/1001.jpg","/image/grimas/015/1001.jpg",,,,,,,,,,,,,"5.45","0.00","0000-00-00 00:00:00","0000-00-00 00:00:00","0000-00-00 00:00:00","0.05","1","2","81","4","0","1","0","2000","0","0","0","0
"
+"3","5","GR010 015 1002","/image/grimas/015/1002.jpg","/image/grimas/015/1002.jpg","/image/grimas/015/1002.jpg",,,,,,,,,,,,,"5.45","0.00","0000-00-00 00:00:00","0000-00-00 00:00:00","0000-00-00 00:00:00","0.05","1","2","81","4","0","0","0","2000","0","0","0","0
"
+
+i78926_input.csv
+"772","3","SPAZZOLA ARIA BABYLISS 32","image/APP.ELET/Varie/Babyliss-Spazzola-Elettrica.jpg","image/APP.ELET/Varie/Babyliss-Spazzola-Elettrica.jpg","image/APP.ELET/Varie/Babyliss-Spazzola-Elettrica.jpg",,,,,,,,,,,,,"33.00","0.00","2006-11-07 09:26:16","2007-05-24 17:54:24",NULL,"1.00","1","2","86","150","0","0","0","2000","0",,,
+"773","1","BIGODINI RISCALDANTI \"BA","image/APP.ELET/Varie/Bigodini-caldi.jpg","image/APP.ELET/Varie/Bigodini-caldi.jpg","image/APP.ELET/Varie/Bigodini-caldi.jpg",,,,,,,,,,,,,"57.91","0.00","2006-11-07 09:26:16","2006-11-07 09:26:16","0000-00-00 00:00:00","0.70","1","2","86","152","0","1","0","2000","0","0","0","0"
+"774","3","PIASTRA \"BABYLISS\" PRO C","image/APP.ELET/piastre/Piastre-Babyliss-2564E.jpg","image/APP.ELET/piastre/Piastre-Babyliss-2564E.jpg","image/APP.ELET/piastre/Piastre-Babyliss-2564E.jpg",,,,,,,,,,,,,"78.00","0.00","2006-11-07 09:26:16","2006-12-13 19:18:31","0000-00-00 00:00:00","0.70","1","2","86","148","0","0","0","2000","0","0","0","0"
+"775","0","PHON ULTRON MAXI 2200W+1","image/APP.ELET/phon/Phon-Ultron-1500W.jpg","image/APP.ELET/phon/Phon-Ultron-1500W.jpg","image/APP.ELET/phon/Phon-Ultron-1500W.jpg",,,,,,,,,,,,,"26.66","0.00","2006-11-07 09:26:16","2006-12-13 18:57:03","0000-00-00 00:00:00","0.70","1","2","101","158","0","0","0","2000","0","0","0","0"
+
+i78926_inconsistent_double_quote_behavior.csv
+Description,Description
+"The following text breaks everything",OpenOffice.org calc doesnt " like quotes embedded in fields.
+Here is some data,Here is some data
+Here is some data,Here is some data
+Here is some data,Here is some data
+Here is some data,Here is some data
+This field will fix everything,Here is the " fix
+First fixed record,Here is some data
+Here is some data,Here is some data
+"This field will break things again",Mismatched quote " must be after some quoted text
+Here is some data,Here is some data
+Here is some data,Here is some data
+Here is some data,Here is some data
+This field will fix everything,Here is the " fix
+First fixed record,Here is some data
+Here is some data,Here is some data
+Here is some data,Here is some data
+Here is some data,Here is some data
+Here is some data,Here is some data
+This next field will /not/ break things because /this/ field is not quoted,Mismatched quote " must be after some quoted text
+Here is some data,Here is some data
+Here is some data,Here is some data
+Here is some data,Here is some data
+
+i80385_test1.csv
+test,"a"b, "a", d"a"c , m , "j" ,d,b"A",D"E,"f,1","a,b","de"b,a"
+"abcdef" test "abc","def"g"h",def"gh"
+"this is
+a test","yes
+it
+works",
+No it doesn't
+"yes it,"","does","no"
+""""a,cd""e
+"a""b""
+",<- needed to end test file here
+
+i80385_test2.csv
+test,"a"b, "a", d"a"c , m , "j" ,d,b"A",D"E,"f,1","a,b","de"b,a
+"abcdef" test "abc","def"g"h",def"gh"
+"this is
+a test","yes
+it
+works",
+No it doesn't
+"yes it,"","does","no"
+""""a,cd""e
+"a""b""
+",<- needed to end test file here
+
+i80385_test3.csv
+test,"a"b, "a", d"a"c , m , "j" ,d,b"A",D"E,"f,1","a,b","de"b,a"
+"abcdef" test "abc","def"g"h",def"gh"
+"this is
+a test","yes
+it
+works",
+No it doesn't
+"yes it,"","does","no"
+""""a,cd""e
+"a""b"""
+
+i80385_test4.csv
+test,"a"b, "a", d"a"c , m , "j" ,d,b"A",D"E,"f,1","a,b","de"b,a
+"abcdef" test "abc","def"g"h",def"gh"
+"this is
+a test","yes
+it
+works",
+No it doesn't
+"yes it,"","does","no"
+""""a,cd""e
+"a""b"""
diff --git a/sc/qa/unit/subsequent_filters-test.cxx b/sc/qa/unit/subsequent_filters-test.cxx
index f2ed04b529f5..f7d5d2b76995 100644
--- a/sc/qa/unit/subsequent_filters-test.cxx
+++ b/sc/qa/unit/subsequent_filters-test.cxx
@@ -52,10 +52,12 @@
 #define ODS_FORMAT_TYPE 50331943
 #define XLS_FORMAT_TYPE 318767171
 #define XLSX_FORMAT_TYPE 268959811
+#define CSV_FORMAT_TYPE 195
 
 #define ODS     0
 #define XLS     1
 #define XLSX    2
+#define CSV     3
 
 using namespace ::com::sun::star;
 using namespace ::com::sun::star::uno;
@@ -69,7 +71,8 @@ struct FileFormat {
 FileFormat aFileFormats[] = {
     { "ods" , "calc8", "", ODS_FORMAT_TYPE },
     { "xls" , "MS Excel 97", "calc_MS_EXCEL_97", XLS_FORMAT_TYPE },
-    { "xlsx", "Calc MS Excel 2007 XML" , "MS Excel 2007 XML", XLSX_FORMAT_TYPE }
+    { "xlsx", "Calc MS Excel 2007 XML" , "MS Excel 2007 XML", XLSX_FORMAT_TYPE },
+    { "csv" , "Text - txt - csv (StarCalc)", "calc_Text_txt_csv_StarCalc", CSV_FORMAT_TYPE }
 };
 
 }
@@ -113,6 +116,7 @@ public:
     void testBugFixesODS();
     void testBugFixesXLS();
     void testBugFixesXLSX();
+    void testBrokenQuotesCSV();
 
     //misc tests unrelated to the import filters
     void testPasswordNew();
@@ -135,6 +139,9 @@ public:
     CPPUNIT_TEST(testBugFixesODS);
     CPPUNIT_TEST(testBugFixesXLS);
     CPPUNIT_TEST(testBugFixesXLSX);
+#if 0
+    CPPUNIT_TEST(testBrokenQuotesCSV);
+#endif
     //disable testPassword on MacOSX due to problems with libsqlite3
     //also crashes on DragonFly due to problems with nss/nspr headers
 #if !defined(MACOSX) && !defined(DRAGONFLY) && !defined(WNT)
@@ -636,6 +643,30 @@ void ScFiltersTest::testBugFixesXLSX()
     xDocSh->DoClose();
 }
 
+void ScFiltersTest::testBrokenQuotesCSV()
+{
+    const rtl::OUString aFileNameBase(RTL_CONSTASCII_USTRINGPARAM("fdo48621_broken_quotes."));
+    rtl::OUString aFileExtension(aFileFormats[CSV].pName, strlen(aFileFormats[CSV].pName), RTL_TEXTENCODING_UTF8 );
+    rtl::OUString aFilterName(aFileFormats[CSV].pFilterName, strlen(aFileFormats[CSV].pFilterName), RTL_TEXTENCODING_UTF8) ;
+    rtl::OUString aFileName;
+    createFileURL(aFileNameBase, aFileExtension, aFileName);
+    rtl::OUString aFilterType(aFileFormats[CSV].pTypeName, strlen(aFileFormats[CSV].pTypeName), RTL_TEXTENCODING_UTF8);
+    std::cout << aFileFormats[CSV].pName << " Test" << std::endl;
+    ScDocShellRef xDocSh = load (aFilterName, aFileName, rtl::OUString(), aFilterType, aFileFormats[CSV].nFormatType);
+
+    CPPUNIT_ASSERT_MESSAGE("Failed to load fdo48621_broken_quotes.csv", xDocSh.Is());
+    ScDocument* pDoc = xDocSh->GetDocument();
+    CPPUNIT_ASSERT_MESSAGE("No Document", pDoc); //remove with first test
+
+    rtl::OUString aSheet2CSV(RTL_CONSTASCII_USTRINGPARAM("fdo48621_broken_quotes_exported."));
+    rtl::OUString aCSVPath;
+    createCSVPath( aSheet2CSV, aCSVPath );
+    // fdo#48621
+    testFile( aCSVPath, pDoc, 0, PureString);
+
+    xDocSh->DoClose();
+}
+
 void ScFiltersTest::testPassword_Impl(const rtl::OUString& aFileNameBase)
 {
     rtl::OUString aFileExtension(aFileFormats[0].pName, strlen(aFileFormats[0].pName), RTL_TEXTENCODING_UTF8 );