Keyang
downstreamFormat - Last in array to contain no comma?
Hi there, trying to read a CSV file and stream its output data to another JSON array file.
var Converter = require("csvtojson").Converter;
var csvConverter = new Converter({
constructResult: false,
downstreamFormat: "array"
});
var readStream = require("fs").createReadStream(req.file.path);
var writeStream = require("fs").createWriteStream("csvData.json");
readStream
.pipe(csvConverter)
.subscribe((jsonObj, index) => {
jsonObj.myNewKey = "some value";
})
.pipe(writeStream);
But the last JSON added to the array adds an extra comma at the end creating an invalid array/json.
As such:
[
{Name: Bob},
{Name: Sarah},
{Name: James},
]
How could I make sure it doesn't add the comma on the last entry? Cheers
Maybe if someone needs a quick solution until it's fixed:
import { Transform } from 'stream';
const csv = csvtojson(csvOptions);
// you can use one more stream to transform csv stream output
const transform = new Transform({
transform (chunk, encoding, callback) {
let string = chunk.toString('utf-8');
if (['[\n', ']\n'].includes(string)) {
if (string === '[\n') {
this.theFirstEntity = true;
}
return callback(null, string);
}
string = string.replace(/,$/gm, '');
if (this.theFirstEntity) {
this.theFirstEntity = false;
} else {
string = `,` + string;
}
callback(null, string);
},
});
readableStream.pipe(csv).pipe(transform).pipe(fs.createWriteStream('output.json'));
/*
output should be the following:
[
{"Name": "Bob"}
,{"Name": "Sarah"}
,{"Name": "James"}
]
*/
I had more luck with this lineToArray transform:
const { Transform } = require('stream');
const csvtojson = require("csvtojson");
const lineToArray = new Transform({
transform (chunk, encoding, cb) {
// add [ to very front
// add , between rows
// remove crlf from row
this.push((this.isNotAtFirstRow ? ',' : '[') + chunk.toString('utf-8').slice(0, -1));
this.isNotAtFirstRow = true;
cb();
},
flush(cb) {
// add ] to very end or [] if no rows
const isEmpty = (!this.isNotAtFirstRow);
this.push(isEmpty ? '[]' : ']');
cb();
}
});
readableStream
.pipe(csvtojson({
checkType: true,
downstreamFormat: 'line'
}))
.pipe(lineToArray)
.pipe(writableStream);
Also #389
Update: Modified to reflect comments on -1 https://github.com/Keyang/node-csvtojson/issues/333#issuecomment-705076249
Maybe if someone needs a quick solution until it's fixed:
import { Transform } from 'stream'; const csv = csvtojson(csvOptions); // you can use one more stream to transform csv stream output const transform = new Transform({ transform (chunk, encoding, callback) { let string = chunk.toString('utf-8'); if (['[\n', ']\n'].includes(string)) { if (string === '[\n') { this.theFirstEntity = true; } return callback(null, string); } string = string.replace(/,$/gm, ''); if (this.theFirstEntity) { this.theFirstEntity = false; } else { string = `,` + string; } callback(null, string); }, }); readableStream.pipe(csv).pipe(transform).pipe(fs.createWriteStream('output.json')); /* output should be the following: [ {"Name": "Bob"} ,{"Name": "Sarah"} ,{"Name": "James"} ] */
hi @ushakov-ruslan Can you please give me a working example as this is not working for me. I need help in fixing comma. @ushakov-ruslan thanks
yes @oliverfoster Please guide If I am doing something wrong.
`parse(){ // you can use one more stream to transform csv stream output const transform = new Transform({ transform (chunk, encoding, callback) { let string = chunk.toString('utf-8');
if (['[\n', ']\n'].includes(string)) {
if (string === '[\n') {
this.theFirstEntity = true;
}
return callback(null, string);
}
string = string.replace(/,$/gm, '');
if (this.theFirstEntity) {
this.theFirstEntity = false;
} else {
string = `,` + string;
}
callback(null, string);
},
});
const self = this;
return new Promise(function(resolve,reject){
const filePath = require('path').resolve(
__dirname+"/../../public/",
self.fileName);
const readStream= fs.createReadStream(filePath);
var stream = fs.createWriteStream(filePath+".json");
readStream.pipe(csv({downstreamFormat: 'array',flatKeys:true,delimiter:"auto",quote:'"',escape:'"',noheader: !self.isHeader,}))
.on('header',(header)=>{
header.map((header)=> header.toString().trim().replace(/[.]/g, ""))
resolve(header);
}).pipe(transform).pipe(stream)
})
}`
You're not using my example. I also couldn't get @ushakov-ruslan s example to work either which is why I left my example. Perhaps the code changed in between?
Mine explicitly switches to the line stream so that each write to the stream is a single JSON item or csv row. It makes everything much simpler to handle.
const lineToArray = new Transform({
transform (chunk, encoding, cb) {
this.push((this.isNotAtFirstRow ? ',' : '[') + chunk.toString('utf-8').slice(0,-2));
this.isNotAtFirstRow = true;
cb();
},
flush(cb) {
const isEmpty = (!this.isNotAtFirstRow);
this.push(isEmpty ? '[]' : ']');
cb();
}
});
const self = this;
return new Promise(function(resolve,reject){
const filePath = require('path').resolve(
__dirname+"/../../public/",
self.fileName);
const readStream= fs.createReadStream(filePath);
var stream = fs.createWriteStream(filePath+".json");
readStream.pipe(csv({downstreamFormat: 'array',flatKeys:true,delimiter:"auto",quote:'"',escape:'"',noheader: !self.isHeader,}))
.on('header',(header)=>{
header.map((header)=> header.toString().trim().replace(/[.]/g, ""))
resolve(header);
}).pipe(lineToArray).pipe(stream)
})
}```
I used this. No array object json created inside the file
downstreamFormat: 'line' No data created in the file.
Also. I want to have array of object.
readStream
.pipe(csv({
downstreamFormat: 'line',
checkType: true
})
.pipe(lineToArray)
.pipe(stream);
Are you debugging? It looks like you're returning a promise from your parse function before creating the stream? It's hard to tell because there's no indentation. https://github.com/adam-p/markdown-here/wiki/Markdown-Cheatsheet#code
I am calling resolve on header to get all header values. The code is working with array of object, The only thing I am getting is an comma at the end
Can you remove all the header stuff and just test it as I wrote it three messages ago? The line parser is not the array parser. Get the array of objects first then work out how to change the headers.
const lineToArray = new Transform({
transform (chunk, encoding, cb) {
this.push((this.isNotAtFirstRow ? ',' : '[') + chunk.toString('utf-8').slice(0,-2));
this.isNotAtFirstRow = true;
cb();
},
flush(cb) {
const isEmpty = (!this.isNotAtFirstRow);
this.push(isEmpty ? '[]' : ']');
cb();
}
});
const self = this;
const filePath = require('path').resolve(
__dirname+"/../../public/",
self.fileName);
const readStream= fs.createReadStream(filePath);
var stream = fs.createWriteStream(filePath+".json");
readStream
.pipe(csv({
downstreamFormat: 'line',
checkType: true
})).pipe(lineToArray).pipe(stream);
}```
I get blank file without any data
Your input filepath isn't right, the CSV isn't formatted correctly, you're not importing the library properly, you're dropping errors in a try catch block around your call to parse or something along those lines. Are you using a debugger? If you're getting an empty file the last line I know is running is the fs.createWriteStream.
Input filepath is correct. CSV is formatted as It is wokring perfect with array of object only getting extra comma at the end. Please check screenshot for readstream data. It is not empty
yes correct. I am using streams.
.pipe(csv({
downstreamFormat: 'line',
checkType: true
})).pipe(lineToArray).pipe(stream);```
What version of node are you using? I want to test myself. It definitely looks like an issue with the way I'm using streams.