SSIS- script task to export data from database table to .csv file
User::INPUT_FILE_PATH - input file path to get output file name same as the input file name
User::OutputPath - output path where we get the output in .csv format
User::outputdatabaseConnectionString - Database connection string
User::INPUT_FILE_PATH - input file path to get output file name same as the input file name
User::OutputPath - output path where we get the output in .csv format
User::outputdatabaseConnectionString - Database connection string
/*
Microsoft SQL Server Integration Services
Script Task
Write scripts using Microsoft Visual C#
2008.
The ScriptMain is the entry point class of
the script.
*/
using System;
using
System.Data;
using
Microsoft.SqlServer.Dts.Runtime;
using
System.Windows.Forms;
using
System.Web;
using
System.Collections.Generic;
//using
System.Linq;
//using
System.Web.Security;
//using
System.Web.UI;
//using
System.Web.UI.WebControls;
//using
System.Web.UI.Adapters;
//using
System.Web.UI.WebControls.WebParts;
//using
System.Web.UI.HtmlControls;
using
System.Data.SqlClient;
using
System.Text;
using System.IO;
namespace
ST_f9e64226a54d4da28a70f42ab090a637.csproj
{
[System.AddIn.AddIn("ScriptMain", Version = "1.0", Publisher = "", Description = "")]
public partial class ScriptMain :
Microsoft.SqlServer.Dts.Tasks.ScriptTask.VSTARTScriptObjectModelBase
{
string
_FileName = "";
string
unicode = "utf-8";
#region VSTA generated code
enum ScriptResults
{
Success =
Microsoft.SqlServer.Dts.Runtime.DTSExecResult.Success,
Failure =
Microsoft.SqlServer.Dts.Runtime.DTSExecResult.Failure
};
#endregion
public void Main()
{
// TODO:
Add your code here
string
path = Dts.Variables["User::INPUT_FILE_PATH"].Value.ToString();
String
fname = Dts.Variables["User::OutputPath"].Value.ToString()
+ "\\" + GetFileNames(path);
Dts.TaskResult = (int)ScriptResults.Success;
//GetFileNames(path);
CreateCSVFile(GetTableData(),
fname);
}
//public
static Encoding GetEncoding(string unicode);
public string GetFileNames(string
path)
{
string
tFileName = new DirectoryInfo(path).GetFiles("*.csv")[0].Name;
//string[]
files = Directory.GetFiles(path);
//string
tfileName=files[0].ToString(); //as assuming always one file
string[]
fileparts = tFileName.Split('_');
string
filePart = fileparts[0] + "_" +
fileparts[1] + "_" + fileparts[2]
+ "_" + "Othertext"
+ "_" + fileparts[5];
_FileName = filePart;
return
_FileName;
}
public DataTable GetTableData()
{
String
connstring = Dts.Variables["User::outputdatabaseConnectionString"].Value.ToString();
SqlConnection
conn = new SqlConnection(connstring);
SqlCommand
cmd = new SqlCommand("select * from table", conn);
cmd.CommandType = CommandType.Text;
SqlDataAdapter
adap = new SqlDataAdapter(cmd);
DataTable
dt = new DataTable();
adap.Fill(dt);
return
dt;
}
public void CreateCSVFile(DataTable
dt, string strFilePath)
{
StreamWriter
sw = new StreamWriter(strFilePath,
false);
int
iColCount = dt.Columns.Count;
for
(int i = 0; i < iColCount; i++)
{
sw.Write(dt.Columns[i]);
if
(i < iColCount - 1)
{
sw.Write(",");
}
}
sw.Write(sw.NewLine);
// Now
write all the rows.
foreach
(DataRow dr in
dt.Rows)
{
for
(int i = 0; i < iColCount; i++)
{
if
(!Convert.IsDBNull(dr[i]))
{
sw.Write(dr[i].ToString());
}
if
(i < iColCount - 1)
{
sw.Write(",");
}
}
sw.Write(sw.NewLine);
}
sw.Close();
}
}
}
I have some weird characters like – . How can I solve code page problem?
ReplyDeleteThanks
Hi Arslan,
DeleteDo you have the – characters in the database? Can you send one sample row for me to understand better.
If i have understood the problem of yours you can use the below line of code in place of the
DeleteStreamWriter sw = new StreamWriter(strFilePath, false);
as
StreamWriter sw = new StreamWriter(new FileStream(strFilePath, FileMode.CreateNew, FileAccess.Write), Encoding.GetEncoding("shift-jis"));
Here "shift-jis" is the encoding pattern for Japan languages like "病院区分". you search for the encoding language of the data you are receiving for and in place of "shift-jis" use that encoding.
Hello Indranil,
ReplyDeleteThank you for your blog, I have searched countless blogs and this is far the easiest example. I was hoping if you could please provide assistance as to where you would put code to cleanup rogue CR LF within the double quotes? Any help appreciated and thank you again
Thanks.
ReplyDeleteAm getting " An exception of type 'System.Data.SqlClient.SqlException' occurred in System.Data.dll but was not handled in user code " error for bigger table.
Also i need to handle null value ( if NULL then NULL and id '' then '')
Can you pls help us for above issue
Hi Thanks for the good representation of the export to CSV... but I tried this and its working well for less data, I have to export some 15 tables which has around 10 Million records each... The package fails throwing some exception from the Script task... is there something been missed for handling this... find below the exception that is received from the script task
System.OutOfMemoryException: Exception of type 'System.OutOfMemoryException' was thrown.
at System.Data.RBTree`1.TreePage..ctor(Int32 size)
at System.Data.RBTree`1.AllocPage(Int32 size)
at System.Data.RBTree`1.GetNewNode(K key)
at System.Data.Index.InitRecords(IFilter filter)
at System.Data.Index..ctor(DataTable table, IndexField[] indexFields, Comparison`1 comparison, DataViewRowState recordStates, IFilter rowFilter)
at System.Data.DataTable.GetIndex(IndexField[] indexDesc, DataViewRowState recordStates, IFilter rowFilter)
at System.Data.DataColumn.get_SortIndex()
at System.Data.DataColumn.IsNotAllowDBNullViolated()
at System.Data.DataTable.EnableConstraints()
at System.Data.DataTable.set_EnforceConstraints(Boolean value)
at System.Data.DataTable.EndLoadData()
at System.Data.Common.DataAdapter.FillFromReader(DataSet dataset, DataTable datatable, String srcTable, DataReaderContainer dataReader, Int32 startRecord, Int32 maxRecords, DataColumn parentChapterColumn, Object parentChapterValue)
at System.Data.Common.DataAdapter.Fill(DataTable[] dataTables, IDataReader dataReader, Int32 startRecord, Int32 maxRecords)
at System.Data.Common.LoadAdapter.FillFromReader(DataTable[] dataTables, IDataReader dataReader, Int32 startRecord, Int32 maxRecords)
at System.Data.DataTable.Load(IDataReader reader, LoadOption loadOption, FillErrorEventHandler errorHandler)
at ST_e687e423204547fc9889e990c3f593da.ScriptMain.Main()
Were you able to fix this "OutOfMemmoryException". I have similar issue while exporting millions of records. Any help Appreciated. thank you.
Delete