MongoDB Reporting

cmd/nuclei/issue-tracker-config.yaml

@@ -162,3 +162,14 @@
 # duplicate-issue-check: false
 # # open-state-id is the ID of the open state in Linear
 # open-state-id: ""
+#mongodb:
+# # the connection string to the MongoDB database
+# # (e.g., mongodb://root:example@localhost:27017/nuclei?ssl=false&authSource=admin)
+# connection-string: ""
+# # the name of the collection to store the issues
+# collection-name: ""
+# # excludes the Request and Response from the results (helps with filesize)
+# omit-raw: false
+# # determines the number of results to be kept in memory before writing it to the database or 0 to
+# # persist all in memory and write all results at the end (default)
+# batch-size: 0

go.mod

@@ -105,6 +105,7 @@ require (
  github.com/stretchr/testify v1.9.0
  github.com/tarunKoyalwar/goleak v0.0.0-20240429141123-0efa90dbdcf9
  github.com/zmap/zgrab2 v0.1.8-0.20230806160807-97ba87c0e706
+ go.mongodb.org/mongo-driver v1.17.0
  golang.org/x/term v0.24.0
  gopkg.in/yaml.v3 v3.0.1
  moul.io/http2curl v1.0.0
@@ -195,6 +196,7 @@ require (
  github.com/mitchellh/mapstructure v1.5.0 // indirect
  github.com/moby/term v0.5.0 // indirect
  github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect
+ github.com/montanaflynn/stats v0.7.1 // indirect
  github.com/muesli/reflow v0.3.0 // indirect
  github.com/muesli/termenv v0.15.3-0.20240618155329-98d742f6907a // indirect
  github.com/opencontainers/go-digest v1.0.0 // indirect
@@ -228,9 +230,13 @@ require (
  github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
  github.com/ugorji/go/codec v1.2.11 // indirect
  github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect
+ github.com/xdg-go/pbkdf2 v1.0.0 // indirect
+ github.com/xdg-go/scram v1.1.2 // indirect
+ github.com/xdg-go/stringprep v1.0.4 // indirect
  github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
  github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
  github.com/xeipuuv/gojsonschema v1.2.0 // indirect
+ github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect
  github.com/ysmood/fetchup v0.2.3 // indirect
  github.com/ysmood/got v0.34.1 // indirect
  github.com/yuin/goldmark v1.7.4 // indirect

go.sum

@@ -755,6 +755,8 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G
 github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
 github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw=
 github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8=
+github.com/montanaflynn/stats v0.7.1 h1:etflOAAHORrCC44V+aR6Ftzort912ZU+YLiSTuV8eaE=
+github.com/montanaflynn/stats v0.7.1/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow=
 github.com/mreiferson/go-httpclient v0.0.0-20160630210159-31f0106b4474/go.mod h1:OQA4XLvDbMgS8P0CevmM4m9Q3Jq4phKUzcocxuGJ5m8=
 github.com/mreiferson/go-httpclient v0.0.0-20201222173833-5e475fde3a4d/go.mod h1:OQA4XLvDbMgS8P0CevmM4m9Q3Jq4phKUzcocxuGJ5m8=
 github.com/muesli/reflow v0.3.0 h1:IFsN6K9NfGtjeggFP+68I4chLZV2yIKsXJFNZ+eWh6s=
@@ -1085,6 +1087,12 @@ github.com/xanzy/go-gitlab v0.107.0 h1:P2CT9Uy9yN9lJo3FLxpMZ4xj6uWcpnigXsjvqJ6nd
 github.com/xanzy/go-gitlab v0.107.0/go.mod h1:wKNKh3GkYDMOsGmnfuX+ITCmDuSDWFO0G+C4AygL9RY=
 github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM=
 github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw=
+github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
+github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
+github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY=
+github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4=
+github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8=
+github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM=
 github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
 github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo=
 github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
@@ -1098,6 +1106,8 @@ github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMx
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
 github.com/yl2chen/cidranger v1.0.2 h1:lbOWZVCG1tCRX4u24kuM1Tb4nHqWkDxwLdoS+SevawU=
 github.com/yl2chen/cidranger v1.0.2/go.mod h1:9U1yz7WPYDwf0vpNWFaeRh0bjwz5RVgRy/9UEQfHl0g=
+github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 h1:ilQV1hzziu+LLM3zUTJ0trRztfwgjqKnBWNtSRkbmwM=
+github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78/go.mod h1:aL8wCCfTfSfmXjznFBSZNN13rSJjlIOI1fUNAtF7rmI=
 github.com/ysmood/fetchup v0.2.3 h1:ulX+SonA0Vma5zUFXtv52Kzip/xe7aj4vqT5AJwQ+ZQ=
 github.com/ysmood/fetchup v0.2.3/go.mod h1:xhibcRKziSvol0H1/pj33dnKrYyI2ebIvz5cOOkYGns=
 github.com/ysmood/goob v0.4.0 h1:HsxXhyLBeGzWXnqVKtmT9qM7EuVs/XOgkX7T6r1o1AQ=
@@ -1150,6 +1160,8 @@ go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
 go.etcd.io/bbolt v1.3.10 h1:+BqfJTcCzTItrop8mq/lbzL8wSGtj94UO/3U31shqG0=
 go.etcd.io/bbolt v1.3.10/go.mod h1:bK3UQLPJZly7IlNmV7uVHJDxfe5aK9Ll93e/74Y9oEQ=
 go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg=
+go.mongodb.org/mongo-driver v1.17.0 h1:Hp4q2MCjvY19ViwimTs00wHi7G4yzxh4/2+nTx8r40k=
+go.mongodb.org/mongo-driver v1.17.0/go.mod h1:wwWm/+BuOddhcq3n68LKRmgk2wXzmF6s0SFOa0GINL4=
 go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=
 go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=

pkg/reporting/exporters/mongo/mongo.go

@@ -0,0 +1,155 @@
+package mongo
+
+import (
+ "context"
+ "github.com/pkg/errors"
+ "github.com/projectdiscovery/gologger"
+ "github.com/projectdiscovery/nuclei/v3/pkg/output"
+ "go.mongodb.org/mongo-driver/mongo"
+ "net/url"
+ "os"
+ "strings"
+ "sync"
+
+ mongooptions "go.mongodb.org/mongo-driver/mongo/options"
+)
+
+type Exporter struct {
+ options *Options
+ mutex *sync.Mutex
+ rows []output.ResultEvent
+ collection *mongo.Collection
+ connection *mongo.Client
+}
+
+// Options contains the configuration options for MongoDB exporter client
+type Options struct {
+ // ConnectionString is the connection string to the MongoDB database
+ ConnectionString string `yaml:"connection-string"`
+ // CollectionName is the name of the MongoDB collection in which to store the results
+ CollectionName string `yaml:"collection-name"`
+ // OmitRaw excludes the Request and Response from the results (helps with filesize)
+ OmitRaw bool `yaml:"omit-raw"`
+ // BatchSize determines the number of results to be kept in memory before writing it to the database or 0 to
+ // persist all in memory and write all results at the end (default)
+ BatchSize int `yaml:"batch-size"`
+}
+
+// New creates a new MongoDB exporter integration client based on options.
+func New(options *Options) (*Exporter, error) {
+ exporter := &Exporter{
+ mutex: &sync.Mutex{},
+ options: options,
+ rows: []output.ResultEvent{},
+ }
+
+ // If the environment variable for the connection string is set, then use that instead. This allows for easier
+ // management of sensitive items such as credentials
+ envConnectionString := os.Getenv("MONGO_CONNECTION_STRING")
+ if envConnectionString != "" {
+ options.ConnectionString = envConnectionString
+ gologger.Info().Msgf("Using connection string from environment variable MONGO_CONNECTION_STRING")
+ }
+
+ // Create the connection to the database
+ clientOptions := mongooptions.Client().ApplyURI(options.ConnectionString)
+
+ // Create a new client and connect to the MongoDB server
+ client, err := mongo.Connect(context.TODO(), clientOptions)
+ if err != nil {
+ gologger.Error().Msgf("Error creating MongoDB client: %s", err)
+ return nil, err
+ }
+
+ // Ensure the connection is valid
+ err = client.Ping(context.Background(), nil)
+ if err != nil {
+ gologger.Error().Msgf("Error connecting to MongoDB: %s", err)
+ return nil, err
+ }
+
+ // Get the database from the connection string to set the database and collection
+ parsed, err := url.Parse(options.ConnectionString)
+ if err != nil {
+ gologger.Error().Msgf("Error parsing connection string: %s", options.ConnectionString)
+ return nil, err
+ }
+
+ databaseName := strings.TrimPrefix(parsed.Path, "/")
+
+ if databaseName == "" {
+ return nil, errors.New("error getting database name from connection string")
+ }
+
+ exporter.connection = client
+ exporter.collection = client.Database(databaseName).Collection(options.CollectionName)
+
+ return exporter, nil
+}
+
+// Export writes a result document to the configured MongoDB collection
+// in the database configured by the connection string
+func (exporter *Exporter) Export(event *output.ResultEvent) error {
+ exporter.mutex.Lock()
+ defer exporter.mutex.Unlock()
+
+ if exporter.options.OmitRaw {
+ event.Request = ""
+ event.Response = ""
+ }
+
+ // Add the row to the queue to be processed
+ exporter.rows = append(exporter.rows, *event)
+
+ // If the batch size is greater than 0 and the number of rows has reached the batch, flush it to the database
+ if exporter.options.BatchSize > 0 && len(exporter.rows) >= exporter.options.BatchSize {
+ err := exporter.WriteRows()
+ if err != nil {
+ // The error is already logged, return it to bubble up to the caller
+ return err
+ }
+ }
+
+ return nil
+}
+
+// WriteRows writes all rows from the rows list to the MongoDB collection and removes them from the list
+func (exporter *Exporter) WriteRows() error {
+ // Loop through the rows and write them, removing them as they're entered
+ for len(exporter.rows) > 0 {
+ data := exporter.rows[0]
+
+ // Write the data to the database
+ _, err := exporter.collection.InsertOne(context.TODO(), data)
+ if err != nil {
+ gologger.Fatal().Msgf("Error inserting record into MongoDB collection: %s", err)
+ return err
+ }
+
+ // Remove the item from the list
+ exporter.rows = exporter.rows[1:]
+ }
+
+ return nil
+}
+
+func (exporter *Exporter) Close() error {
+ exporter.mutex.Lock()
+ defer exporter.mutex.Unlock()
+
+ // Write all pending rows
+ err := exporter.WriteRows()
+ if err != nil {
+ // The error is already logged, return it to bubble up to the caller
+ return err
+ }
+
+ // Close the database connection
+ err = exporter.connection.Disconnect(context.TODO())
+ if err != nil {
+ gologger.Error().Msgf("Error disconnecting from MongoDB: %s", err)
+ return err
+ }
+
+ return nil
+}

pkg/reporting/options.go

@@ -5,6 +5,7 @@ import (
  "github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/jsonexporter"
  "github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/jsonl"
  "github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/markdown"
+ "github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/mongo"
  "github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/sarif"
  "github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/splunk"
  "github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/filters"
@@ -44,6 +45,8 @@ type Options struct {
  JSONExporter *jsonexporter.Options `yaml:"json"`
  // JSONLExporter contains configuration options for JSONL Exporter Module
  JSONLExporter *jsonl.Options `yaml:"jsonl"`
+ // MongoDBExporter containers the configuration options for the MongoDB Exporter Module
+ MongoDBExporter *mongo.Options `yaml:"mongodb"`
 
  HttpClient *retryablehttp.Client `yaml:"-"`
  OmitRaw bool `yaml:"-"`

pkg/reporting/reporting.go

@@ -2,6 +2,7 @@ package reporting
 
 import (
  "fmt"
+ "github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/mongo"
  "os"
  "strings"
  "sync/atomic"
@@ -166,6 +167,13 @@ func New(options *Options, db string, doNotDedupe bool) (Client, error) {
  }
  client.exporters = append(client.exporters, exporter)
  }
+ if options.MongoDBExporter != nil {
+ exporter, err := mongo.New(options.MongoDBExporter)
+ if err != nil {
+ return nil, errorutil.NewWithErr(err).Wrap(ErrExportClientCreation)
+ }
+ client.exporters = append(client.exporters, exporter)
+ }
 
  if doNotDedupe {
  return client, nil
@@ -212,6 +220,7 @@ func CreateConfigIfNotExists() error {
  SplunkExporter: &splunk.Options{},
  JSONExporter: &json_exporter.Options{},
  JSONLExporter: &jsonl.Options{},
+ MongoDBExporter: &mongo.Options{},
  }
  reportingFile, err := os.Create(reportingConfig)
  if err != nil {