Add a new NDJSON / JSONL input source (trufflesecurity#4721)

This adds a new input source to TruffleHog, accessible via `trufflehog json-enumerator`.

This input source requires a list of filenames, each of which is an NDJSON-formatted sequence of objects that take one of two forms:

Form 1: `{"data": "utf-8 string", "metadata": <non-null JSON value>}`
Form 2: `{"data_b64": "base64-encoded bytestring", "metadata": <non-null JSON value>}`

The `data` / `data_b64` field specifies the content to be scanned. The `metadata` field is arbitrary, and is simply propagated downstream with scan results from the corresponding content.

Note that although `trufflehog json-enumerator` requires a list of filenames to be given, the NDJSON data that you wish to scan may not need to be first written to disk. On Linux and macOS, at least, you can use shell process substitution to set up a named pipe from a producer process, like `trufflehog json-enumerator <(some-program-that-emits-ndjson)`.

Author: bradlarsen

main.go

@@ -270,6 +270,9 @@ var (
 	stdinInputScan = cli.Command("stdin", "Find credentials from stdin.")
 	multiScanScan  = cli.Command("multi-scan", "Find credentials in multiple sources defined in configuration.")
 
+	jsonEnumeratorScan  = cli.Command("json-enumerator", "Find credentials from a JSON enumerator input.")
+	jsonEnumeratorPaths = jsonEnumeratorScan.Arg("path", "Path to JSON enumerator file to scan.").Strings()
+
 	analyzeCmd = analyzer.Command(cli)
 	usingTUI   = false
 )
@@ -1114,6 +1117,13 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics,
 		} else {
 			refs = []sources.JobProgressRef{ref}
 		}
+	case jsonEnumeratorScan.FullCommand():
+		cfg := sources.JSONEnumeratorConfig{Paths: *jsonEnumeratorPaths}
+		if ref, err := eng.ScanJSONEnumeratorInput(ctx, cfg); err != nil {
+			return scanMetrics, fmt.Errorf("failed to scan JSON enumerator input: %v", err)
+		} else {
+			refs = []sources.JobProgressRef{ref}
+		}
 	default:
 		return scanMetrics, fmt.Errorf("invalid command: %s", cmd)
 	}

pkg/engine/json_enumerator.go

@@ -0,0 +1,43 @@
+package engine
+
+import (
+	"runtime"
+
+	"google.golang.org/protobuf/proto"
+	"google.golang.org/protobuf/types/known/anypb"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/sources/json_enumerator"
+)
+
+// ScanJSONEnumeratorInput scans input that is in JSON Enumerator format
+func (e *Engine) ScanJSONEnumeratorInput(
+	ctx context.Context,
+	c sources.JSONEnumeratorConfig,
+) (sources.JobProgressRef, error) {
+	connection := &sourcespb.JSONEnumerator{
+		Paths: c.Paths,
+	}
+	var conn anypb.Any
+	err := anypb.MarshalFrom(&conn, connection, proto.MarshalOptions{})
+	if err != nil {
+		ctx.Logger().Error(err, "failed to marshal JSON enumerator connection")
+		return sources.JobProgressRef{}, err
+	}
+
+	sourceName := "trufflehog - JSON enumerator"
+	sourceID, jobID, err := e.sourceManager.GetIDs(ctx, sourceName, json_enumerator.SourceType)
+	if err != nil {
+		ctx.Logger().Error(err, "failed to get IDs from source manager")
+		return sources.JobProgressRef{}, err
+	}
+
+	source := &json_enumerator.Source{}
+	err = source.Init(ctx, sourceName, jobID, sourceID, true, &conn, runtime.NumCPU())
+	if err != nil {
+		return sources.JobProgressRef{}, err
+	}
+	return e.sourceManager.EnumerateAndScan(ctx, sourceName, source)
+}