From 197ad12908993e20975f05a1d72b62a0339d1528 Mon Sep 17 00:00:00 2001 From: milisarge Date: Wed, 28 Jun 2017 00:46:51 +0300 Subject: [PATCH] json-araci --- bin/json | 1684 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 1684 insertions(+) create mode 100755 bin/json diff --git a/bin/json b/bin/json new file mode 100755 index 000000000..89a9559f6 --- /dev/null +++ b/bin/json @@ -0,0 +1,1684 @@ +#!/usr/bin/env node +/** + * Copyright (c) 2014 Trent Mick. All rights reserved. + * Copyright (c) 2014 Joyent Inc. All rights reserved. + * + * json -- JSON love for your command line. + * + * See and + */ + +var VERSION = '9.0.6'; + +var p = console.warn; +var util = require('util'); +var assert = require('assert'); +var path = require('path'); +var vm = require('vm'); +var fs = require('fs'); +var warn = console.warn; +var EventEmitter = require('events').EventEmitter; + + + +//--- exports for module usage + +exports.main = main; +exports.getVersion = getVersion; +exports.parseLookup = parseLookup; + +// As an exported API, these are still experimental: +exports.lookupDatum = lookupDatum; +exports.printDatum = printDatum; // DEPRECATED + + + +//---- globals and constants + +// Output modes. +var OM_JSONY = 1; +var OM_JSON = 2; +var OM_INSPECT = 3; +var OM_COMPACT = 4; +var OM_FROM_NAME = { + 'jsony': OM_JSONY, + 'json': OM_JSON, + 'inspect': OM_INSPECT, + 'compact': OM_COMPACT +}; + + + +//---- support functions + +function getVersion() { + return VERSION; +} + +/** + * Return a *shallow* copy of the given object. + * + * Only support objects that you get out of JSON, i.e. no functions. + */ +function objCopy(obj) { + var copy; + if (Array.isArray(obj)) { + copy = obj.slice(); + } else if (typeof (obj) === 'object') { + copy = {}; + Object.keys(obj).forEach(function (k) { + copy[k] = obj[k]; + }); + } else { + copy = obj; // immutable type + } + return copy; +} + +if (util.format) { + format = util.format; +} else { + // From : + var formatRegExp = /%[sdj%]/g; + + function format(f) { + var i; + if (typeof (f) !== 'string') { + var objects = []; + for (i = 0; i < arguments.length; i++) { + objects.push(util.inspect(arguments[i])); + } + return objects.join(' '); + } + i = 1; + var args = arguments; + var len = args.length; + var str = String(f).replace(formatRegExp, function (x) { + if (i >= len) + return x; + switch (x) { + case '%s': + return String(args[i++]); + case '%d': + return Number(args[i++]); + case '%j': + return JSON.stringify(args[i++]); + case '%%': + return '%'; + default: + return x; + } + }); + for (var x = args[i]; i < len; x = args[++i]) { + if (x === null || typeof (x) !== 'object') { + str += ' ' + x; + } else { + str += ' ' + util.inspect(x); + } + } + return str; + } +} + +/** + * Parse the given string into a JS string. Basically: handle escapes. + */ +function _parseString(s) { + /* JSSTYLED */ + var quoted = '"' + s.replace(/\\"/, '"').replace('"', '\\"') + '"'; + return eval(quoted); +} + +// json_parse.js () +/* BEGIN JSSTYLED */ +// START json_parse +var json_parse=function(){"use strict";var a,b,c={'"':'"',"\\":"\\","/":"/",b:"\b",f:"\f",n:"\n",r:"\r",t:"\t"},d,e=function(b){throw{name:"SyntaxError",message:b,at:a,text:d}},f=function(c){return c&&c!==b&&e("Expected '"+c+"' instead of '"+b+"'"),b=d.charAt(a),a+=1,b},g=function(){var a,c="";b==="-"&&(c="-",f("-"));while(b>="0"&&b<="9")c+=b,f();if(b==="."){c+=".";while(f()&&b>="0"&&b<="9")c+=b}if(b==="e"||b==="E"){c+=b,f();if(b==="-"||b==="+")c+=b,f();while(b>="0"&&b<="9")c+=b,f()}a=+c;if(!isFinite(a))e("Bad number");else return a},h=function(){var a,d,g="",h;if(b==='"')while(f()){if(b==='"')return f(),g;if(b==="\\"){f();if(b==="u"){h=0;for(d=0;d<4;d+=1){a=parseInt(f(),16);if(!isFinite(a))break;h=h*16+a}g+=String.fromCharCode(h)}else if(typeof c[b]=="string")g+=c[b];else break}else g+=b}e("Bad string")},i=function(){while(b&&b<=" ")f()},j=function(){switch(b){case"t":return f("t"),f("r"),f("u"),f("e"),!0;case"f":return f("f"),f("a"),f("l"),f("s"),f("e"),!1;case"n":return f("n"),f("u"),f("l"),f("l"),null}e("Unexpected '"+b+"'")},k,l=function(){var a=[];if(b==="["){f("["),i();if(b==="]")return f("]"),a;while(b){a.push(k()),i();if(b==="]")return f("]"),a;f(","),i()}}e("Bad array")},m=function(){var a,c={};if(b==="{"){f("{"),i();if(b==="}")return f("}"),c;while(b){a=h(),i(),f(":"),Object.hasOwnProperty.call(c,a)&&e('Duplicate key "'+a+'"'),c[a]=k(),i();if(b==="}")return f("}"),c;f(","),i()}}e("Bad object")};return k=function(){i();switch(b){case"{":return m();case"[":return l();case'"':return h();case"-":return g();default:return b>="0"&&b<="9"?g():j()}},function(c,f){var g;return d=c,a=0,b=" ",g=k(),i(),b&&e("Syntax error"),typeof f=="function"?function h(a,b){var c,d,e=a[b];if(e&&typeof e=="object")for(c in e)Object.prototype.hasOwnProperty.call(e,c)&&(d=h(e,c),d!==undefined?e[c]=d:delete e[c]);return f.call(a,b,e)}({"":g},""):g}}(); +// END json_parse +/* END JSSTYLED */ + +function printHelp() { + /* BEGIN JSSTYLED */ + var w = console.log; + w('Usage:'); + w(' | json [OPTIONS] [LOOKUPS...]'); + w(' json -f FILE [OPTIONS] [LOOKUPS...]'); + w(''); + w('Pipe in your JSON for pretty-printing, JSON validation, filtering, '); + w('and modification. Supply one or more `LOOKUPS` to extract a '); + w('subset of the JSON. HTTP header blocks are skipped by default.'); + w('Roughly in order of processing, features are:'); + w(''); + w('Grouping:'); + w(' Use "-g" or "--group" to group adjacent objects, separated by'); + w(' by no space or a by a newline, or adjacent arrays, separate by'); + w(' by a newline. This can be helpful for, e.g.: '); + w(' $ cat *.json | json -g ... '); + w(' and similar.'); + w(''); + w('Execution:'); + w(' Use the "-e CODE" option to execute JavaScript code on the input JSON.'); + w(' $ echo \'{"name":"trent","age":38}\' | json -e \'this.age++\''); + w(' {'); + w(' "name": "trent",'); + w(' "age": 39'); + w(' }'); + w(' If input is an array, this will automatically process each'); + w(' item separately.'); + w(''); + w('Conditional filtering:'); + w(' Use the "-c CODE" option to filter the input JSON.'); + w(' $ echo \'[{"age":38},{"age":4}]\' | json -c \'this.age>21\''); + w(' [{\'age\':38}]'); + w(' If input is an array, this will automatically process each'); + w(' item separately. Note: "CODE" is JavaScript code.'); + w(''); + w('Lookups:'); + w(' Use lookup arguments to extract particular values:'); + w(' $ echo \'{"name":"trent","age":38}\' | json name'); + w(' trent'); + w(''); + w(' Use "-a" for *array processing* of lookups and *tabular output*:'); + w(' $ echo \'{"name":"trent","age":38}\' | json name age'); + w(' trent'); + w(' 38'); + w(' $ echo \'[{"name":"trent","age":38},'); + w(' {"name":"ewan","age":4}]\' | json -a name age'); + w(' trent 38'); + w(' ewan 4'); + w(''); + w('In-place editing:'); + w(' Use "-I, --in-place" to edit a file in place:'); + w(' $ json -I -f config.json # reformat'); + w(' $ json -I -f config.json -c \'this.logLevel="debug"\' # add field'); + w(''); + w('Pretty-printing:'); + w(' Output is "jsony" by default: 2-space indented JSON, except a'); + w(' single string value is printed without quotes.'); + w(' $ echo \'{"name": "trent", "age": 38}\' | json'); + w(' {'); + w(' "name": "trent",'); + w(' "age": 38'); + w(' }'); + w(' $ echo \'{"name": "trent", "age": 38}\' | json name'); + w(' trent'); + w(''); + w(" Use '-j' or '-o json' for explicit JSON, '-o json-N' for N-space indent:"); + w(' $ echo \'{"name": "trent", "age": 38}\' | json -o json-0'); + w(' {"name":"trent","age":38}'); + w(''); + w('Options:'); + w(' -h, --help Print this help info and exit.'); + w(' --version Print version of this command and exit.'); + w(' -q, --quiet Don\'t warn if input isn\'t valid JSON.'); + w(''); + w(' -f FILE Path to a file to process. If not given, then'); + w(' stdin is used.'); + w(' -I, --in-place In-place edit of the file given with "-f".'); + w(' Lookups are not allow with in-place editing'); + w(' because it makes it too easy to lose content.'); + w(''); + w(' -H Drop any HTTP header block (as from `curl -i ...`).'); + w(' -g, --group Group adjacent objects or arrays into an array.'); + w(' --merge Merge adjacent objects into one. Keys in last '); + w(' object win.'); + w(' --deep-merge Same as "--merge", but will recurse into objects '); + w(' under the same key in both.') + w(' -a, --array Process input as an array of separate inputs'); + w(' and output in tabular form.'); + w(' -A Process input as a single object, i.e. stop'); + w(' "-e" and "-c" automatically processing each'); + w(' item of an input array.'); + w(' -d DELIM Delimiter char for tabular output (default is " ").'); + w(' -D DELIM Delimiter char between lookups (default is "."). E.g.:'); + w(' $ echo \'{"a.b": {"b": 1}}\' | json -D / a.b/b'); + w(''); + w(' -M, --items Itemize an object into an array of '); + w(' {"key": , "value": }'); + w(' objects for easier processing.'); + w(''); + w(' -e CODE Execute the given JavaScript code on the input. If input'); + w(' is an array, then each item of the array is processed'); + w(' separately (use "-A" to override).'); + w(' -c CODE Filter the input with JavaScript `CODE`. If `CODE`'); + w(' returns false-y, then the item is filtered out. If'); + w(' input is an array, then each item of the array is '); + w(' processed separately (use "-A" to override).'); + w(''); + w(' -k, --keys Output the input object\'s keys.'); + w(' -n, --validate Just validate the input (no processing or output).'); + w(' Use with "-q" for silent validation (exit status).'); + w(''); + w(' -o, --output MODE'); + w(' Specify an output mode. One of:'); + w(' jsony (default): JSON with string quotes elided'); + w(' json: JSON output, 2-space indent'); + w(' json-N: JSON output, N-space indent, e.g. "json-4"'); + w(' inspect: node.js `util.inspect` output'); + w(' -i Shortcut for `-o inspect`'); + w(' -j Shortcut for `-o json`'); + w(' -0, -2, -4 Set indentation to the given value w/o setting MODE.'); + w(' -0 => -o jsony-0'); + w(' -4 => -o jsony-4'); + w(' -j0 => -o json-0'); + w(''); + w('See for more docs and '); + w(' for project details.'); + /* END JSSTYLED */ +} + + +/** + * Parse the command-line options and arguments into an object. + * + * { + * 'args': [...] // arguments + * 'help': true, // true if '-h' option given + * // etc. + * } + * + * @return {Object} The parsed options. `.args` is the argument list. + * @throws {Error} If there is an error parsing argv. + */ +function parseArgv(argv) { + var parsed = { + args: [], + help: false, + quiet: false, + dropHeaders: false, + exeSnippets: [], + condSnippets: [], + outputMode: OM_JSONY, + jsonIndent: 2, + array: null, + delim: ' ', + lookupDelim: '.', + items: false, + outputKeys: false, + group: false, + merge: null, // --merge -> 'shallow', --deep-merge -> 'deep' + inputFiles: [], + validate: false, + inPlace: false + }; + + // Turn '-iH' into '-i -H', except for argument-accepting options. + var args = argv.slice(2); // drop ['node', 'scriptname'] + var newArgs = []; + var optTakesArg = { + 'd': true, + 'o': true, + 'D': true + }; + for (var i = 0; i < args.length; i++) { + if (args[i] === '--') { + newArgs = newArgs.concat(args.slice(i)); + break; + } + if (args[i].charAt(0) === '-' && args[i].charAt(1) !== '-' && + args[i].length > 2) + { + var splitOpts = args[i].slice(1).split(''); + for (var j = 0; j < splitOpts.length; j++) { + newArgs.push('-' + splitOpts[j]) + if (optTakesArg[splitOpts[j]]) { + var optArg = splitOpts.slice(j + 1).join(''); + if (optArg.length) { + newArgs.push(optArg); + } + break; + } + } + } else { + newArgs.push(args[i]); + } + } + args = newArgs; + + endOfOptions = false; + while (args.length > 0) { + var arg = args.shift(); + if (endOfOptions) { + parsed.args.push(arg); + break; + } + switch (arg) { + case '--': + endOfOptions = true; + break; + case '-h': // display help and exit + case '--help': + parsed.help = true; + break; + case '--version': + parsed.version = true; + break; + case '-q': + case '--quiet': + parsed.quiet = true; + break; + case '-H': // drop any headers + parsed.dropHeaders = true; + break; + case '-o': + case '--output': + var name = args.shift(); + if (!name) { + throw new Error('no argument given for "-o|--output" option'); + } + var idx = name.lastIndexOf('-'); + if (idx !== -1) { + var indent = name.slice(idx + 1); + if (/^\d+$/.test(indent)) { + parsed.jsonIndent = Number(indent); + name = name.slice(0, idx); + } else if (indent === 'tab') { + parsed.jsonIndent = '\t'; + name = name.slice(0, idx); + } + } + parsed.outputMode = OM_FROM_NAME[name]; + if (parsed.outputMode === undefined) { + throw new Error('unknown output mode: "' + name + '"'); + } + break; + case '-0': + parsed.jsonIndent = 0; + break; + case '-2': + parsed.jsonIndent = 2; + break; + case '-4': + parsed.jsonIndent = 4; + break; + case '-I': + case '--in-place': + parsed.inPlace = true; + break; + case '-i': // output with util.inspect + parsed.outputMode = OM_INSPECT; + break; + case '-j': // output with JSON.stringify + parsed.outputMode = OM_JSON; + break; + case '-a': + case '--array': + parsed.array = true; + break; + case '-A': + parsed.array = false; + break; + case '-d': + parsed.delim = _parseString(args.shift()); + break; + case '-D': + parsed.lookupDelim = args.shift(); + if (parsed.lookupDelim.length !== 1) { + throw new Error(format( + 'invalid lookup delim "%s" (must be a single char)', + parsed.lookupDelim)); + } + break; + case '-e': + case '-E': // DEPRECATED in v9 + parsed.exeSnippets.push(args.shift()); + break; + case '-c': + case '-C': // DEPRECATED in v9 + parsed.condSnippets.push(args.shift()); + break; + case '-M': + case '--items': + parsed.items = true; + break; + case '-k': + case '--keys': + parsed.outputKeys = true; + break; + case '-g': + case '--group': + parsed.group = true; + break; + case '--merge': + parsed.merge = 'shallow'; + break; + case '--deep-merge': + parsed.merge = 'deep'; + break; + case '-f': + parsed.inputFiles.push(args.shift()); + break; + case '-n': + case '--validate': + parsed.validate = true; + break; + default: // arguments + if (!endOfOptions && arg.length > 0 && arg[0] === '-') { + throw new Error('unknown option "' + arg + '"'); + } + parsed.args.push(arg); + break; + } + } + + if (parsed.group && parsed.merge) { + throw new Error('cannot use -g|--group and --merge options together'); + } + if (parsed.outputKeys && parsed.args.length > 0) { + throw new Error( + 'cannot use -k|--keys option and lookup arguments together'); + } + if (parsed.inPlace && parsed.inputFiles.length !== 1) { + throw new Error('must specify exactly one file with "-f FILE" to ' + + 'use -I/--in-place'); + } + if (parsed.inPlace && parsed.args.length > 0) { + throw new Error('lookups cannot be specified with in-place editing ' + + '(-I/--in-place), too easy to lose content'); + } + + return parsed; +} + + + +/** + * Streams chunks from given file paths or stdin. + * + * @param opts {Object} Parsed options. + * @returns {Object} An emitter that emits 'chunk', 'error', and 'end'. + * - `emit('chunk', chunk, [obj])` where chunk is a complete block of JSON + * ready to parse. If `obj` is provided, it is the already parsed + * JSON. + * - `emit('error', error)` when an underlying stream emits an error + * - `emit('end')` when all streams are done + */ +function chunkEmitter(opts) { + var emitter = new EventEmitter(); + var streaming = true; + var chunks = []; + var leftover = ''; + var finishedHeaders = false; + + function stripHeaders(s) { + // Take off a leading HTTP header if any and pass it through. + while (true) { + if (s.slice(0, 5) === 'HTTP/') { + var index = s.indexOf('\r\n\r\n'); + var sepLen = 4; + if (index == -1) { + index = s.indexOf('\n\n'); + sepLen = 2; + } + if (index != -1) { + if (!opts.dropHeaders) { + emit(s.slice(0, index + sepLen)); + } + var is100Continue = ( + s.slice(0, 21) === 'HTTP/1.1 100 Continue'); + s = s.slice(index + sepLen); + if (is100Continue) { + continue; + } + finishedHeaders = true; + } + } else { + finishedHeaders = true; + } + break; + } + //console.warn('stripHeaders done, finishedHeaders=%s', finishedHeaders) + return s; + } + + function emitChunks(block, emitter) { + //console.warn('emitChunks start: block="%s"', block) + /* JSSTYLED */ + var splitter = /(})(\s*\n\s*)?({\s*")/; + var leftTrimmedBlock = block.trimLeft(); + if (leftTrimmedBlock && leftTrimmedBlock[0] !== '{') { + // Currently only support streaming consecutive *objects*. + streaming = false; + chunks.push(block); + return ''; + } + /** + * Example: + * > '{"a":"b"}\n{"a":"b"}\n{"a":"b"}'.split(/(})(\s*\n\s*)?({\s*")/) + * [ '{"a":"b"', + * '}', + * '\n', + * '{"', + * 'a":"b"', + * '}', + * '\n', + * '{"', + * 'a":"b"}' ] + */ + var bits = block.split(splitter); + //console.warn('emitChunks: bits (length %d): %j', bits.length, bits); + if (bits.length === 1) { + /* + * An unwanted side-effect of using a regex to find + * newline-separated objects *with a regex*, is that we are looking + * for the end of one object leading into the start of a another. + * That means that we can end up buffering a complete object until + * a subsequent one comes in. If the input stream has large delays + * between objects, then this is unwanted buffering. + * + * One solution would be full stream parsing of objects a la + * . This would nicely + * also remove the artibrary requirement that the input stream be + * newline separated. jsonparse apparently has some issues tho, so + * I don't want to use it right now. It also isn't *small* so not + * sure I want to inline it (`json` doesn't have external deps). + * + * An alternative: The block we have so far one of: + * 1. some JSON that we don't support grouping (e.g. a stream of + * non-objects), + * 2. a JSON object fragment, or + * 3. a complete JSON object (with a possible trailing '{') + * + * If #3, then we can just emit this as a chunk right now. + * + * TODO(PERF): Try out avoiding the first more complete regex split + * for a presumed common case of single-line newline-separated JSON + * objects (e.g. a bunyan log). + */ + // An object must end with '}'. This is an early out to avoid + // `JSON.parse` which I'm *presuming* is slower. + var trimmed = block.split(/\s*\r?\n/)[0]; + if (trimmed[trimmed.length - 1] === '}') { + var obj; + try { + obj = JSON.parse(block); + } catch (e) { + /* pass through */ + } + if (obj !== undefined) { + // Emit the parsed `obj` to avoid re-parsing it later. + emitter.emit('chunk', block, obj); + block = ''; + } + } + return block; + } else { + var n = bits.length - 2; + var s; + s = bits[0] + bits[1]; + emitter.emit('chunk', s, JSON.parse(s)); + for (var i = 3; i < n; i += 4) { + s = bits[i] + bits[i + 1] + bits[i + 2]; + emitter.emit('chunk', s, JSON.parse(s)); + } + return bits[n] + bits[n + 1]; + } + } + + function addDataListener(stream) { + stream.on('data', function (chunk) { + var s = leftover + chunk; + if (!finishedHeaders) { + s = stripHeaders(s); + } + if (!finishedHeaders) { + leftover = s; + } else { + if (!streaming) { + chunks.push(chunk); + return; + } + if (chunk.lastIndexOf('\n') >= 0) { + leftover = emitChunks(s, emitter); + } else { + leftover = s; + } + } + }); + } + + if (opts.inputFiles.length > 0) { + // Stream each file in order. + var i = 0; + + function addErrorListener(file) { + file.on('error', function (err) { + emitter.emit( + 'error', + format('could not read "%s": %s', opts.inputFiles[i], e) + ); + }); + } + + function addEndListener(file) { + file.on('end', function () { + if (i < opts.inputFiles.length) { + var next = opts.inputFiles[i++]; + var nextFile = fs.createReadStream(next, + {encoding: 'utf8'}); + addErrorListener(nextFile); + addEndListener(nextFile); + addDataListener(nextFile); + } else { + if (!streaming) { + emitter.emit('chunk', chunks.join('')); + } else if (leftover) { + leftover = emitChunks(leftover, emitter); + emitter.emit('chunk', leftover); + } + emitter.emit('end'); + } + }); + } + var first = fs.createReadStream(opts.inputFiles[i++], + {encoding: 'utf8'}); + addErrorListener(first); + addEndListener(first); + addDataListener(first); + } else { + // Streaming from stdin. + var stdin = process.openStdin(); + stdin.setEncoding('utf8'); + addDataListener(stdin); + stdin.on('end', function () { + if (!streaming) { + emitter.emit('chunk', chunks.join('')); + } else if (leftover) { + leftover = emitChunks(leftover, emitter); + emitter.emit('chunk', leftover); + } + emitter.emit('end'); + }); + } + return emitter; +} + +/** + * Get input from either given file paths or stdin. If `opts.inPlace` then + * this calls the callback once for each `opts.inputFiles`. + * + * @param opts {Object} Parsed options. + * @param callback {Function} `function (err, content, filename)` where err + * is an error string if there was a problem, `content` is the read + * content and `filename` is the associated file name from which content + * was loaded if applicable. + */ +function getInput(opts, callback) { + if (opts.inputFiles.length === 0) { + // Read from stdin. + var chunks = []; + + var stdin = process.openStdin(); + stdin.setEncoding('utf8'); + stdin.on('data', function (chunk) { + chunks.push(chunk); + }); + + stdin.on('end', function () { + callback(null, chunks.join('')); + }); + } else if (opts.inPlace) { + for (var i = 0; i < opts.inputFiles.length; i++) { + var file = opts.inputFiles[i]; + var content; + try { + content = fs.readFileSync(file, 'utf8'); + } catch (e) { + callback(e, null, file); + } + if (content) { + callback(null, content, file); + } + } + } else { + // Read input files. + var i = 0; + var chunks = []; + try { + for (; i < opts.inputFiles.length; i++) { + chunks.push(fs.readFileSync(opts.inputFiles[i], 'utf8')); + } + } catch (e) { + return callback( + format('could not read "%s": %s', opts.inputFiles[i], e)); + } + callback(null, chunks.join(''), + (opts.inputFiles.length === 1 ? opts.inputFiles[0] : undefined)); + } +} + + +function isInteger(s) { + return (s.search(/^-?[0-9]+$/) == 0); +} + + +/** + * Parse a lookup string into a list of lookup bits. E.g.: + * + * 'a.b.c' -> ["a","b","c"] + * 'b["a"]' -> ["b","a"] + * 'b["a" + "c"]' -> ["b","ac"] + * + * Optionally receives an alternative lookup delimiter (other than '.') + */ +function parseLookup(lookup, lookupDelim) { + var debug = function () {}; + //var debug = console.warn; + + var bits = []; + debug('\n*** ' + lookup + ' ***'); + + bits = []; + lookupDelim = lookupDelim || '.'; + var bit = ''; + var states = [null]; + var escaped = false; + var ch = null; + for (var i = 0; i < lookup.length; ++i) { + var escaped = (!escaped && ch === '\\'); + var ch = lookup[i]; + debug('-- i=' + i + ', ch=' + JSON.stringify(ch) + ' escaped=' + + JSON.stringify(escaped)); + debug('states: ' + JSON.stringify(states)); + + if (escaped) { + bit += ch; + continue; + } + + switch (states[states.length - 1]) { + case null: + switch (ch) { + case '"': + case '\'': + states.push(ch); + bit += ch; + break; + case '[': + states.push(ch); + if (bit !== '') { + bits.push(bit); + bit = '' + } + bit += ch; + break; + case lookupDelim: + if (bit !== '') { + bits.push(bit); + bit = '' + } + break; + default: + bit += ch; + break; + } + break; + + case '[': + bit += ch; + switch (ch) { + case '"': + case '\'': + case '[': + states.push(ch); + break; + case ']': + states.pop(); + if (states[states.length - 1] === null) { + var evaled = vm.runInNewContext( + '(' + bit.slice(1, -1) + ')', {}, ''); + bits.push(evaled); + bit = '' + } + break; + } + break; + + case '"': + bit += ch; + switch (ch) { + case '"': + states.pop(); + if (states[states.length - 1] === null) { + bits.push(bit); + bit = '' + } + break; + } + break; + + case '\'': + bit += ch; + switch (ch) { + case '\'': + states.pop(); + if (states[states.length - 1] === null) { + bits.push(bit); + bit = '' + } + break; + } + break; + } + debug('bit: ' + JSON.stringify(bit)); + debug('bits: ' + JSON.stringify(bits)); + } + + if (bit !== '') { + bits.push(bit); + bit = '' + } + + // Negative-intify: strings that are negative ints we change to a Number for + // special handling in `lookupDatum`: Python-style negative array indexing. + var negIntPat = /^-\d+$/; + for (var i = 0; i < bits.length; i++) { + if (negIntPat.test(bits[i])) { + bits[i] = Number(bits[i]); + } + } + + debug(JSON.stringify(lookup) + ' -> ' + JSON.stringify(bits)); + return bits +} + + +/** + * Parse the given stdin input into: + * { + * 'error': ... error object if there was an error ..., + * 'datum': ... parsed object if content was JSON ... + * } + * + * @param buffer {String} The text to parse as JSON. + * @param obj {Object} Optional. Set when in streaming mode to avoid + * re-interpretation of `group`. Also avoids reparsing. + * @param group {Boolean} Default false. If true, then non-JSON input + * will be attempted to be 'arrayified' (see inline comment). + * @param merge {Boolean} Default null. Can be 'shallow' or 'deep'. An + * attempt will be made to interpret the input as adjacent objects to + * be merged, last key wins. See inline comment for limitations. + */ +function parseInput(buffer, obj, group, merge) { + if (obj) { + return { + datum: obj + }; + } else if (group) { + /** + * Special case: Grouping (previously called auto-arrayification) + * of unjoined list of objects: + * {"one": 1}{"two": 2} + * and auto-concatenation of unjoined list of arrays: + * ["a", "b"]["c", "d"] + * + * This can be nice to process a stream of JSON objects generated from + * multiple calls to another tool or `cat *.json | json`. + * + * Rules: + * - Only JS objects and arrays. Don't see strong need for basic + * JS types right now and this limitation simplifies. + * - The break between JS objects has to include a newline: + * {"one": 1} + * {"two": 2} + * or no spaces at all: + * {"one": 1}{"two": 2} + * I.e., not this: + * {"one": 1} {"two": 2} + * This condition should be fine for typical use cases and ensures + * no false matches inside JS strings. + * - The break between JS *arrays* has to include a newline: + * ["one", "two"] + * ["three"] + * The 'no spaces' case is NOT supported for JS arrays as of v6.0.0 + * because shows that that + * is not safe. + */ + var newBuffer = buffer; + /* JSSTYLED */ + [/(})\s*\n\s*({)/g, /(})({")/g].forEach(function (pat) { + newBuffer = newBuffer.replace(pat, '$1,\n$2'); + }); + [/(\])\s*\n\s*(\[)/g].forEach(function (pat) { + newBuffer = newBuffer.replace(pat, ',\n'); + }); + newBuffer = newBuffer.trim(); + if (newBuffer[0] !== '[') { + newBuffer = '[\n' + newBuffer; + } + if (newBuffer.slice(-1) !== ']') { + newBuffer = newBuffer + '\n]\n'; + } + try { + return { + datum: JSON.parse(newBuffer) + }; + } catch (e2) { + return { + error: e2 + }; + } + } else if (merge) { + // See the 'Rules' above for limitations on boundaries for 'adjacent' + // objects: KISS. + var newBuffer = buffer; + /* JSSTYLED */ + [/(})\s*\n\s*({)/g, /(})({")/g].forEach(function (pat) { + newBuffer = newBuffer.replace(pat, '$1,\n$2'); + }); + newBuffer = '[\n' + newBuffer + '\n]\n'; + var objs; + try { + objs = JSON.parse(newBuffer); + } catch (e) { + return { + error: e + }; + } + var merged = objs[0]; + if (merge === 'shallow') { + for (var i = 1; i < objs.length; i++) { + var obj = objs[i]; + Object.keys(obj).forEach(function (k) { + merged[k] = obj[k]; + }); + } + } else if (merge === 'deep') { + function deepExtend(a, b) { + Object.keys(b).forEach(function (k) { + if (a[k] && b[k] && + toString.call(a[k]) === '[object Object]' && + toString.call(b[k]) === '[object Object]') + { + deepExtend(a[k], b[k]) + } else { + a[k] = b[k]; + } + }); + } + for (var i = 1; i < objs.length; i++) { + deepExtend(merged, objs[i]); + } + } else { + throw new Error(format('unknown value for "merge": "%s"', merge)); + } + return { + datum: merged + }; + } else { + try { + return { + datum: JSON.parse(buffer) + }; + } catch (e) { + return { + error: e + }; + } + } +} + + +/** + * Apply a lookup to the given datum. + * + * @argument datum {Object} + * @argument lookup {Array} The parsed lookup (from + * `parseLookup(, )`). Might be empty. + * @returns {Object} The result of the lookup. + */ +function lookupDatum(datum, lookup) { + var d = datum; + for (var i = 0; i < lookup.length; i++) { + var bit = lookup[i]; + if (d === null) { + return undefined; + } else if (typeof (bit) === 'number' && bit < 0) { + d = d[d.length + bit]; + } else { + d = d[bit]; + } + if (d === undefined) { + return undefined; + } + } + return d; +} + + +/** + * Output the given datasets. + * + * @param datasets {Array} Array of data sets to print, in the form: + * `[ [, , ], ... ]` + * @param filename {String} The filename to which to write the output. If + * not set, then emit to stdout. + * @param headers {String} The HTTP header block string, if any, to emit + * first. + * @param opts {Object} Parsed tool options. + */ +function printDatasets(datasets, filename, headers, opts) { + var isTTY = (filename ? false : process.stdout.isTTY) + var write = emit; + if (filename) { + var tmpPath = path.resolve(path.dirname(filename), + format('.%s-json-%s-%s.tmp', path.basename(filename), process.pid, + Date.now())); + var stats = fs.statSync(filename); + var f = fs.createWriteStream(tmpPath, + {encoding: 'utf8', mode: stats.mode}); + write = f.write.bind(f); + } + if (headers && headers.length > 0) { + write(headers) + } + for (var i = 0; i < datasets.length; i++) { + var dataset = datasets[i]; + var output = stringifyDatum(dataset[0], opts, isTTY); + var sep = dataset[1]; + if (output && output.length) { + write(output); + write(sep); + } else if (dataset[2]) { + write(sep); + } + } + if (filename) { + f.on('open', function () { + f.end(); + fs.renameSync(tmpPath, filename); + if (!opts.quiet) { + warn('json: updated "%s" in-place', filename); + } + }); + } +} + + +/** + * Stringify the given datum according to the given output options. + */ +function stringifyDatum(datum, opts, isTTY) { + var output = null; + switch (opts.outputMode) { + case OM_INSPECT: + output = util.inspect(datum, false, Infinity, isTTY); + break; + case OM_JSON: + if (typeof (datum) !== 'undefined') { + output = JSON.stringify(datum, null, opts.jsonIndent); + } + break; + case OM_COMPACT: + // Dev Note: A still relatively experimental attempt at a more + // compact ouput somewhat a la Python's repr of a dict. I.e. try to + // fit elements on one line as much as reasonable. + if (datum === undefined) { + // pass + } else if (Array.isArray(datum)) { + var bits = ['[\n']; + datum.forEach(function (d) { + bits.push(' ') + bits.push(JSON.stringify(d, null, 0).replace( + /* JSSTYLED */ + /,"(?![,:])/g, ', "')); + bits.push(',\n'); + }); + bits.push(bits.pop().slice(0, -2) + '\n') // drop last comma + bits.push(']'); + output = bits.join(''); + } else { + output = JSON.stringify(datum, null, 0); + } + break; + case OM_JSONY: + if (typeof (datum) === 'string') { + output = datum; + } else if (typeof (datum) !== 'undefined') { + output = JSON.stringify(datum, null, opts.jsonIndent); + } + break; + default: + throw new Error('unknown output mode: ' + opts.outputMode); + } + return output; +} + + +/** + * Print out a single result, considering input options. + * + * @deprecated + */ +function printDatum(datum, opts, sep, alwaysPrintSep) { + var output = stringifyDatum(datum, opts); + if (output && output.length) { + emit(output); + emit(sep); + } else if (alwaysPrintSep) { + emit(sep); + } +} + + +var stdoutFlushed = true; +function emit(s) { + // TODO:PERF If this is try/catch is too slow (too granular): move up to + // mainline and be sure to only catch this particular error. + if (drainingStdout) { + return; + } + try { + stdoutFlushed = process.stdout.write(s); + } catch (e) { + // Handle any exceptions in stdout writing in the 'error' event above. + } +} + +process.stdout.on('error', function (err) { + if (err.code === 'EPIPE') { + // See . + drainStdoutAndExit(0); + } else { + warn(err) + drainStdoutAndExit(1); + } +}); + + +/** + * A hacked up version of 'process.exit' that will first drain stdout + * before exiting. *WARNING: This doesn't stop event processing.* IOW, + * callers have to be careful that code following this call isn't + * accidentally executed. + * + * In node v0.6 "process.stdout and process.stderr are blocking when they + * refer to regular files or TTY file descriptors." However, this hack might + * still be necessary in a shell pipeline. + */ +var drainingStdout = false; +function drainStdoutAndExit(code) { + if (drainingStdout) { + return; + } + drainingStdout = true; + process.stdout.on('drain', function () { + process.exit(code); + }); + process.stdout.on('close', function () { + process.exit(code); + }); + if (stdoutFlushed) { + process.exit(code); + } +} + + +/** + * Return a function for the given JS code that returns. + * + * If no 'return' in the given javascript snippet, then assume we are a single + * statement and wrap in 'return (...)'. This is for convenience for short + * '-c ...' snippets. + */ +function funcWithReturnFromSnippet(js) { + // auto-"return" + if (js.indexOf('return') === -1) { + if (js.substring(js.length - 1) === ';') { + js = js.substring(0, js.length - 1); + } + js = 'return (' + js + ')'; + } + return (new Function(js)); +} + + + +//---- mainline + +function main(argv) { + var opts; + try { + opts = parseArgv(argv); + } catch (e) { + warn('json: error: %s', e.message) + return drainStdoutAndExit(1); + } + //warn(opts); + if (opts.help) { + printHelp(); + return; + } + if (opts.version) { + if (opts.outputMode === OM_JSON) { + var v = { + version: getVersion(), + author: 'Trent Mick', + project: 'https://github.com/trentm/json' + }; + console.log(JSON.stringify(v, null, opts.jsonIndent)); + } else { + console.log('json ' + getVersion()); + console.log('written by Trent Mick'); + console.log('https://github.com/trentm/json'); + } + return; + } + var lookupStrs = opts.args; + + // Prepare condition and execution funcs (and vm scripts) for -c/-e. + var execVm = Boolean(process.env.JSON_EXEC && + process.env.JSON_EXEC === 'vm'); + var i; + var condFuncs = []; + if (!execVm) { + for (i = 0; i < opts.condSnippets.length; i++) { + condFuncs[i] = funcWithReturnFromSnippet(opts.condSnippets[i]); + } + } + var condScripts = []; + if (execVm) { + for (i = 0; i < opts.condSnippets.length; i++) { + condScripts[i] = vm.createScript(opts.condSnippets[i]); + } + } + var cond = Boolean(condFuncs.length + condScripts.length); + var exeFuncs = []; + if (!execVm) { + for (i = 0; i < opts.exeSnippets.length; i++) { + exeFuncs[i] = new Function(opts.exeSnippets[i]); + } + } + var exeScripts = []; + if (execVm) { + for (i = 0; i < opts.exeSnippets.length; i++) { + exeScripts[i] = vm.createScript(opts.exeSnippets[i]); + } + } + var exe = Boolean(exeFuncs.length + exeScripts.length); + + var lookups = lookupStrs.map(function (lookup) { + return parseLookup(lookup, opts.lookupDelim); + }); + + if (opts.group && opts.array && opts.outputMode !== OM_JSON) { + // streaming + var chunker = chunkEmitter(opts); + chunker.on('error', function (error) { + warn('json: error: %s', err.message); + return drainStdoutAndExit(1); + }); + chunker.on('chunk', parseChunk); + } else if (opts.inPlace) { + assert.equal(opts.inputFiles.length, 1, + 'cannot handle more than one file with -I'); + getInput(opts, function (err, content, filename) { + if (err) { + warn('json: error: %s', err.message) + return drainStdoutAndExit(1); + } + + // Take off a leading HTTP header if any and pass it through. + var headers = []; + while (true) { + if (content.slice(0, 5) === 'HTTP/') { + var index = content.indexOf('\r\n\r\n'); + var sepLen = 4; + if (index == -1) { + index = content.indexOf('\n\n'); + sepLen = 2; + } + if (index != -1) { + if (!opts.dropHeaders) { + headers.push(content.slice(0, index + sepLen)); + } + var is100Continue = ( + content.slice(0, 21) === 'HTTP/1.1 100 Continue'); + content = content.slice(index + sepLen); + if (is100Continue) { + continue; + } + } + } + break; + } + parseChunk(content, undefined, filename, true, headers.join('')); + }); + } else { + // not streaming + getInput(opts, function (err, buffer, filename) { + if (err) { + warn('json: error: %s', err.message) + return drainStdoutAndExit(1); + } + // Take off a leading HTTP header if any and pass it through. + while (true) { + if (buffer.slice(0, 5) === 'HTTP/') { + var index = buffer.indexOf('\r\n\r\n'); + var sepLen = 4; + if (index == -1) { + index = buffer.indexOf('\n\n'); + sepLen = 2; + } + if (index != -1) { + if (!opts.dropHeaders) { + emit(buffer.slice(0, index + sepLen)); + } + var is100Continue = ( + buffer.slice(0, 21) === 'HTTP/1.1 100 Continue'); + buffer = buffer.slice(index + sepLen); + if (is100Continue) { + continue; + } + } + } + break; + } + parseChunk(buffer, null, filename, false); + }); + } + + /** + * Parse a single chunk of JSON. This may be called more than once + * (when streaming or when operating on multiple files). + * + * @param chunk {String} The JSON-encoded string. + * @param obj {Object} Optional. For some code paths while streaming `obj` + * will be provided. This is an already parsed JSON object. + * @param filename {String} Optional. The filename from which this content + * came, if relevant. + * @param inPlace {Boolean} Optional. If true, then output will be written + * to `filename`. + * @param headers {String} Optional. Leading HTTP headers, if any to emit. + */ + function parseChunk(chunk, obj, filename, inPlace, headers) { + // Expect the chunk to be JSON. + if (!chunk.length) { + return; + } + // parseInput() -> {datum: , error: } + var input = parseInput(chunk, obj, opts.group, opts.merge); + if (input.error) { + // Doesn't look like JSON. Just print it out and move on. + if (!opts.quiet) { + // Use JSON-js' "json_parse" parser to get more detail on the + // syntax error. + var details = ''; + var normBuffer = chunk.replace(/\r\n|\n|\r/, '\n'); + try { + json_parse(normBuffer); + details = input.error; + } catch (err) { + // err.at has the position. Get line/column from that. + var at = err.at - 1; // `err.at` looks to be 1-based. + var lines = chunk.split('\n'); + var line, col, pos = 0; + for (line = 0; line < lines.length; line++) { + pos += lines[line].length + 1; + if (pos > at) { + col = at - (pos - lines[line].length - 1); + break; + } + } + var spaces = ''; + for (var i = 0; i < col; i++) { + spaces += '.'; + } + details = err.message + ' at line ' + (line + 1) + + ', column ' + (col + 1) + ':\n ' + + lines[line] + '\n ' + spaces + '^'; + } + warn('json: error: %s is not JSON: %s', + filename ? '"' + filename + '"' : 'input', details); + } + if (!opts.validate) { + emit(chunk); + if (chunk.length && chunk[chunk.length - 1] !== '\n') { + emit('\n'); + } + } + return drainStdoutAndExit(1); + } + if (opts.validate) { + return drainStdoutAndExit(0); + } + var data = input.datum; + + // Process: items (-M, --items) + if (opts.items) { + if (!Array.isArray(data)) { + var key; + var array = []; + for (key in data) { + if (data.hasOwnProperty(key)) { + array.push({ + key: key, + value: data[key] + }); + } + } + data = array; + } + } + + // Process: executions (-e, -E) + var i, j; + if (!exe) { + /* pass */ + } else if (opts.array || (opts.array === null && Array.isArray(data))) { + var arrayified = false; + if (!Array.isArray(data)) { + arrayified = true; + data = [data]; + } + for (i = 0; i < data.length; i++) { + var datum = data[i]; + for (j = 0; j < exeFuncs.length; j++) { + exeFuncs[j].call(datum); + } + for (j = 0; j < exeScripts.length; j++) { + exeScripts[j].runInNewContext(datum); + } + } + if (arrayified) { + data = data[0]; + } + } else { + for (j = 0; j < exeFuncs.length; j++) { + exeFuncs[j].call(data); + } + for (j = 0; j < exeScripts.length; j++) { + exeScripts[j].runInNewContext(data); + } + } + + // Process: conditionals (-c) + if (!cond) { + /* pass */ + } else if (opts.array || (opts.array === null && Array.isArray(data))) { + var arrayified = false; + if (!Array.isArray(data)) { + arrayified = true; + data = [data]; + } + var filtered = []; + for (i = 0; i < data.length; i++) { + var datum = data[i]; + var datumCopy = objCopy(datum); + var keep = true; + // TODO(perf): Perhaps drop the 'datumCopy'? "this is a gun" + for (j = 0; j < condFuncs.length; j++) { + if (!condFuncs[j].call(datumCopy)) { + keep = false; + break; + } + } + if (keep) { + for (j = 0; j < condScripts.length; j++) { + if (!condScripts[j].runInNewContext(datumCopy)) { + keep = false; + break; + } + } + if (keep) { + filtered.push(datum); + } + } + } + if (arrayified) { + data = (filtered.length ? filtered[0] : []); + } else { + data = filtered; + } + } else { + var keep = true; + var dataCopy = objCopy(data); + for (j = 0; j < condFuncs.length; j++) { + // TODO(perf): Perhaps drop the 'dataCopy'? "this is a gun" + if (!condFuncs[j].call(dataCopy)) { + keep = false; + break; + } + } + if (keep) { + for (j = 0; j < condScripts.length; j++) { + if (!condScripts[j].runInNewContext(dataCopy)) { + keep = false; + break; + } + } + } + if (!keep) { + data = undefined; + } + } + + // Process: lookups + var lookupsAreIndeces = false; + if (lookups.length) { + if (opts.array) { + if (!Array.isArray(data)) data = [data]; + var table = []; + for (j = 0; j < data.length; j++) { + var datum = data[j]; + var row = {}; + for (i = 0; i < lookups.length; i++) { + var lookup = lookups[i]; + var value = lookupDatum(datum, lookup); + if (value !== undefined) { + row[lookup.join('.')] = value; + } + } + table.push(row); + } + data = table; + } else { + // Special case handling: Note if the 'lookups' are indeces into + // an array. This may be used below to change the output + // representation. + if (Array.isArray(data)) { + lookupsAreIndeces = true; + for (i = 0; i < lookups.length; i++) { + if (lookups[i].length !== 1 || + isNaN(Number(lookups[i]))) + { + lookupsAreIndeces = false; + break; + } + } + } + var row = {}; + for (i = 0; i < lookups.length; i++) { + var lookup = lookups[i]; + var value = lookupDatum(data, lookup); + if (value !== undefined) { + row[lookup.join('.')] = value; + } + } + data = row; + } + } + + // --keys + if (opts.outputKeys) { + var data = Object.keys(data); + } + + // Output + var datasets = []; + if (opts.outputMode === OM_JSON) { + if (lookups.length === 1 && !opts.array) { + /** + * Special case: For JSON output of a *single* lookup, *don't* + * use the full table structure, else there is no way to get + * string quoting for a single value: + * $ echo '{"a": [], "b": "[]"}' | json -j a + * [] + * $ echo '{"a": [], "b": "[]"}' | json -j b + * '[]' + * See for why. + */ + data = data[lookups[0].join('.')]; + } else if (lookupsAreIndeces) { + /** + * Special case: Lookups that are all indeces into an input + * array are more likely to be wanted as an array of selected + * items rather than a 'JSON table' thing that we use otherwise. + */ + var flattened = []; + for (i = 0; i < lookups.length; i++) { + var lookupStr = lookups[i].join('.'); + if (data.hasOwnProperty(lookupStr)) { + flattened.push(data[lookupStr]) + } + } + data = flattened; + } + // If JSON output mode, then always just output full set of data to + // ensure valid JSON output. + datasets.push([data, '\n', false]); + } else if (lookups.length) { + if (opts.array) { + // Output `data` as a 'table' of lookup results. + for (j = 0; j < data.length; j++) { + var row = data[j]; + for (i = 0; i < lookups.length - 1; i++) { + datasets.push([row[lookups[i].join('.')], + opts.delim, true]); + } + datasets.push([row[lookups[i].join('.')], '\n', true]); + } + } else { + for (i = 0; i < lookups.length; i++) { + datasets.push([data[lookups[i].join('.')], '\n', false]); + } + } + } else if (opts.array) { + if (!Array.isArray(data)) data = [data]; + for (j = 0; j < data.length; j++) { + datasets.push([data[j], '\n', false]); + } + } else { + // Output `data` as is. + datasets.push([data, '\n', false]); + } + printDatasets(datasets, inPlace ? filename : undefined, headers, opts); + } +} + +if (require.main === module) { + // HACK guard for . + // We override the `process.stdout.end` guard that core node.js puts in + // place. The real fix is that `.end()` shouldn't be called on stdout + // in node core. Hopefully node v0.6.9 will fix that. Only guard + // for v0.6.0..v0.6.8. + var nodeVer = process.versions.node.split('.').map(Number); + if ([0, 6, 0] <= nodeVer && nodeVer <= [0, 6, 8]) { + var stdout = process.stdout; + stdout.end = stdout.destroy = stdout.destroySoon = function () { + /* pass */ + }; + } + + main(process.argv); +}