From 2d615875277f78ae0a710c09b74155df654c46c7 Mon Sep 17 00:00:00 2001
From: Kashif Khan <kashif.khan@trufflesec.com>
Date: Fri, 27 Mar 2026 15:26:42 +0500
Subject: [PATCH 01/20] basic structure for source

---
 pkg/pb/sourcespb/sources.pb.go          | 424 +++++++++++++++---------
 pkg/pb/sourcespb/sources.pb.validate.go | 104 ++++++
 pkg/sources/sources.go                  |  16 +
 pkg/sources/web/web.go                  |  42 +++
 proto/sources.proto                     |   8 +
 5 files changed, 429 insertions(+), 165 deletions(-)
 create mode 100644 pkg/sources/web/web.go

diff --git a/pkg/pb/sourcespb/sources.pb.go b/pkg/pb/sourcespb/sources.pb.go
index 37424d7eee7c..323b3483b981 100644
--- a/pkg/pb/sourcespb/sources.pb.go
+++ b/pkg/pb/sourcespb/sources.pb.go
@@ -71,6 +71,7 @@ const (
 	SourceType_SOURCE_TYPE_STDIN                      SourceType = 40
 	SourceType_SOURCE_TYPE_SLACK_CONTINUOUS           SourceType = 41
 	SourceType_SOURCE_TYPE_JSON_ENUMERATOR            SourceType = 42
+	SourceType_SOURCE_TYPE_WEB                        SourceType = 43
 )
 
 // Enum value maps for SourceType.
@@ -119,6 +120,7 @@ var (
 		40: "SOURCE_TYPE_STDIN",
 		41: "SOURCE_TYPE_SLACK_CONTINUOUS",
 		42: "SOURCE_TYPE_JSON_ENUMERATOR",
+		43: "SOURCE_TYPE_WEB",
 	}
 	SourceType_value = map[string]int32{
 		"SOURCE_TYPE_AZURE_STORAGE":              0,
@@ -164,6 +166,7 @@ var (
 		"SOURCE_TYPE_STDIN":                      40,
 		"SOURCE_TYPE_SLACK_CONTINUOUS":           41,
 		"SOURCE_TYPE_JSON_ENUMERATOR":            42,
+		"SOURCE_TYPE_WEB":                        43,
 	}
 )
 
@@ -5100,6 +5103,77 @@ func (x *JSONEnumerator) GetPaths() []string {
 	return nil
 }
 
+type Web struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	Urls  []string `protobuf:"bytes,1,rep,name=urls,proto3" json:"urls,omitempty"`
+	Crawl bool     `protobuf:"varint,2,opt,name=crawl,proto3" json:"crawl,omitempty"`
+	Depth int64    `protobuf:"varint,3,opt,name=depth,proto3" json:"depth,omitempty"`
+	Delay int64    `protobuf:"varint,4,opt,name=delay,proto3" json:"delay,omitempty"`
+}
+
+func (x *Web) Reset() {
+	*x = Web{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_sources_proto_msgTypes[41]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *Web) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*Web) ProtoMessage() {}
+
+func (x *Web) ProtoReflect() protoreflect.Message {
+	mi := &file_sources_proto_msgTypes[41]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use Web.ProtoReflect.Descriptor instead.
+func (*Web) Descriptor() ([]byte, []int) {
+	return file_sources_proto_rawDescGZIP(), []int{41}
+}
+
+func (x *Web) GetUrls() []string {
+	if x != nil {
+		return x.Urls
+	}
+	return nil
+}
+
+func (x *Web) GetCrawl() bool {
+	if x != nil {
+		return x.Crawl
+	}
+	return false
+}
+
+func (x *Web) GetDepth() int64 {
+	if x != nil {
+		return x.Depth
+	}
+	return 0
+}
+
+func (x *Web) GetDelay() int64 {
+	if x != nil {
+		return x.Delay
+	}
+	return 0
+}
+
 var File_sources_proto protoreflect.FileDescriptor
 
 var file_sources_proto_rawDesc = []byte{
@@ -5892,101 +5966,108 @@ var file_sources_proto_rawDesc = []byte{
 	0x74, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x70, 0x72, 0x6f, 0x6a,
 	0x65, 0x63, 0x74, 0x49, 0x64, 0x22, 0x26, 0x0a, 0x0e, 0x4a, 0x53, 0x4f, 0x4e, 0x45, 0x6e, 0x75,
 	0x6d, 0x65, 0x72, 0x61, 0x74, 0x6f, 0x72, 0x12, 0x14, 0x0a, 0x05, 0x70, 0x61, 0x74, 0x68, 0x73,
-	0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x05, 0x70, 0x61, 0x74, 0x68, 0x73, 0x2a, 0xc4, 0x09,
-	0x0a, 0x0a, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x1d, 0x0a, 0x19,
-	0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x41, 0x5a, 0x55, 0x52,
-	0x45, 0x5f, 0x53, 0x54, 0x4f, 0x52, 0x41, 0x47, 0x45, 0x10, 0x00, 0x12, 0x19, 0x0a, 0x15, 0x53,
-	0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x42, 0x49, 0x54, 0x42, 0x55,
-	0x43, 0x4b, 0x45, 0x54, 0x10, 0x01, 0x12, 0x18, 0x0a, 0x14, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45,
-	0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x43, 0x49, 0x52, 0x43, 0x4c, 0x45, 0x43, 0x49, 0x10, 0x02,
-	0x12, 0x1a, 0x0a, 0x16, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f,
-	0x43, 0x4f, 0x4e, 0x46, 0x4c, 0x55, 0x45, 0x4e, 0x43, 0x45, 0x10, 0x03, 0x12, 0x16, 0x0a, 0x12,
-	0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x44, 0x4f, 0x43, 0x4b,
-	0x45, 0x52, 0x10, 0x04, 0x12, 0x13, 0x0a, 0x0f, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54,
-	0x59, 0x50, 0x45, 0x5f, 0x45, 0x43, 0x52, 0x10, 0x05, 0x12, 0x13, 0x0a, 0x0f, 0x53, 0x4f, 0x55,
-	0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x43, 0x53, 0x10, 0x06, 0x12, 0x16,
-	0x0a, 0x12, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x49,
-	0x54, 0x48, 0x55, 0x42, 0x10, 0x07, 0x12, 0x1a, 0x0a, 0x16, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45,
-	0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x50, 0x55, 0x42, 0x4c, 0x49, 0x43, 0x5f, 0x47, 0x49, 0x54,
-	0x10, 0x08, 0x12, 0x16, 0x0a, 0x12, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50,
-	0x45, 0x5f, 0x47, 0x49, 0x54, 0x4c, 0x41, 0x42, 0x10, 0x09, 0x12, 0x14, 0x0a, 0x10, 0x53, 0x4f,
-	0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4a, 0x49, 0x52, 0x41, 0x10, 0x0a,
-	0x12, 0x24, 0x0a, 0x20, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f,
-	0x4e, 0x50, 0x4d, 0x5f, 0x55, 0x4e, 0x41, 0x55, 0x54, 0x48, 0x44, 0x5f, 0x50, 0x41, 0x43, 0x4b,
-	0x41, 0x47, 0x45, 0x53, 0x10, 0x0b, 0x12, 0x25, 0x0a, 0x21, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45,
-	0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x50, 0x59, 0x50, 0x49, 0x5f, 0x55, 0x4e, 0x41, 0x55, 0x54,
-	0x48, 0x44, 0x5f, 0x50, 0x41, 0x43, 0x4b, 0x41, 0x47, 0x45, 0x53, 0x10, 0x0c, 0x12, 0x12, 0x0a,
-	0x0e, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x33, 0x10,
-	0x0d, 0x12, 0x15, 0x0a, 0x11, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45,
-	0x5f, 0x53, 0x4c, 0x41, 0x43, 0x4b, 0x10, 0x0e, 0x12, 0x1a, 0x0a, 0x16, 0x53, 0x4f, 0x55, 0x52,
-	0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x46, 0x49, 0x4c, 0x45, 0x53, 0x59, 0x53, 0x54,
-	0x45, 0x4d, 0x10, 0x0f, 0x12, 0x13, 0x0a, 0x0f, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54,
-	0x59, 0x50, 0x45, 0x5f, 0x47, 0x49, 0x54, 0x10, 0x10, 0x12, 0x14, 0x0a, 0x10, 0x53, 0x4f, 0x55,
-	0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x54, 0x45, 0x53, 0x54, 0x10, 0x11, 0x12,
-	0x1b, 0x0a, 0x17, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53,
-	0x33, 0x5f, 0x55, 0x4e, 0x41, 0x55, 0x54, 0x48, 0x45, 0x44, 0x10, 0x12, 0x12, 0x2a, 0x0a, 0x26,
-	0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x49, 0x54, 0x48,
-	0x55, 0x42, 0x5f, 0x55, 0x4e, 0x41, 0x55, 0x54, 0x48, 0x45, 0x4e, 0x54, 0x49, 0x43, 0x41, 0x54,
-	0x45, 0x44, 0x5f, 0x4f, 0x52, 0x47, 0x10, 0x13, 0x12, 0x19, 0x0a, 0x15, 0x53, 0x4f, 0x55, 0x52,
-	0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x42, 0x55, 0x49, 0x4c, 0x44, 0x4b, 0x49, 0x54,
-	0x45, 0x10, 0x14, 0x12, 0x16, 0x0a, 0x12, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59,
-	0x50, 0x45, 0x5f, 0x47, 0x45, 0x52, 0x52, 0x49, 0x54, 0x10, 0x15, 0x12, 0x17, 0x0a, 0x13, 0x53,
-	0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4a, 0x45, 0x4e, 0x4b, 0x49,
-	0x4e, 0x53, 0x10, 0x16, 0x12, 0x15, 0x0a, 0x11, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54,
-	0x59, 0x50, 0x45, 0x5f, 0x54, 0x45, 0x41, 0x4d, 0x53, 0x10, 0x17, 0x12, 0x21, 0x0a, 0x1d, 0x53,
-	0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4a, 0x46, 0x52, 0x4f, 0x47,
-	0x5f, 0x41, 0x52, 0x54, 0x49, 0x46, 0x41, 0x43, 0x54, 0x4f, 0x52, 0x59, 0x10, 0x18, 0x12, 0x16,
-	0x0a, 0x12, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x59,
-	0x53, 0x4c, 0x4f, 0x47, 0x10, 0x19, 0x12, 0x27, 0x0a, 0x23, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45,
-	0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x50, 0x55, 0x42, 0x4c, 0x49, 0x43, 0x5f, 0x45, 0x56, 0x45,
-	0x4e, 0x54, 0x5f, 0x4d, 0x4f, 0x4e, 0x49, 0x54, 0x4f, 0x52, 0x49, 0x4e, 0x47, 0x10, 0x1a, 0x12,
-	0x1e, 0x0a, 0x1a, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53,
-	0x4c, 0x41, 0x43, 0x4b, 0x5f, 0x52, 0x45, 0x41, 0x4c, 0x54, 0x49, 0x4d, 0x45, 0x10, 0x1b, 0x12,
-	0x1c, 0x0a, 0x18, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47,
-	0x4f, 0x4f, 0x47, 0x4c, 0x45, 0x5f, 0x44, 0x52, 0x49, 0x56, 0x45, 0x10, 0x1c, 0x12, 0x1a, 0x0a,
-	0x16, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x48, 0x41,
-	0x52, 0x45, 0x50, 0x4f, 0x49, 0x4e, 0x54, 0x10, 0x1d, 0x12, 0x1c, 0x0a, 0x18, 0x53, 0x4f, 0x55,
-	0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x43, 0x53, 0x5f, 0x55, 0x4e, 0x41,
-	0x55, 0x54, 0x48, 0x45, 0x44, 0x10, 0x1e, 0x12, 0x1b, 0x0a, 0x17, 0x53, 0x4f, 0x55, 0x52, 0x43,
-	0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x41, 0x5a, 0x55, 0x52, 0x45, 0x5f, 0x52, 0x45, 0x50,
-	0x4f, 0x53, 0x10, 0x1f, 0x12, 0x18, 0x0a, 0x14, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54,
-	0x59, 0x50, 0x45, 0x5f, 0x54, 0x52, 0x41, 0x56, 0x49, 0x53, 0x43, 0x49, 0x10, 0x20, 0x12, 0x17,
-	0x0a, 0x13, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x50, 0x4f,
-	0x53, 0x54, 0x4d, 0x41, 0x4e, 0x10, 0x21, 0x12, 0x17, 0x0a, 0x13, 0x53, 0x4f, 0x55, 0x52, 0x43,
-	0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x57, 0x45, 0x42, 0x48, 0x4f, 0x4f, 0x4b, 0x10, 0x22,
-	0x12, 0x1d, 0x0a, 0x19, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f,
-	0x45, 0x4c, 0x41, 0x53, 0x54, 0x49, 0x43, 0x53, 0x45, 0x41, 0x52, 0x43, 0x48, 0x10, 0x23, 0x12,
-	0x1b, 0x0a, 0x17, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x48,
-	0x55, 0x47, 0x47, 0x49, 0x4e, 0x47, 0x46, 0x41, 0x43, 0x45, 0x10, 0x24, 0x12, 0x23, 0x0a, 0x1f,
-	0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x49, 0x54, 0x48,
-	0x55, 0x42, 0x5f, 0x45, 0x58, 0x50, 0x45, 0x52, 0x49, 0x4d, 0x45, 0x4e, 0x54, 0x41, 0x4c, 0x10,
-	0x25, 0x12, 0x16, 0x0a, 0x12, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45,
-	0x5f, 0x53, 0x45, 0x4e, 0x54, 0x52, 0x59, 0x10, 0x26, 0x12, 0x1f, 0x0a, 0x1b, 0x53, 0x4f, 0x55,
+	0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x05, 0x70, 0x61, 0x74, 0x68, 0x73, 0x22, 0x5b, 0x0a,
+	0x03, 0x57, 0x65, 0x62, 0x12, 0x12, 0x0a, 0x04, 0x75, 0x72, 0x6c, 0x73, 0x18, 0x01, 0x20, 0x03,
+	0x28, 0x09, 0x52, 0x04, 0x75, 0x72, 0x6c, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x63, 0x72, 0x61, 0x77,
+	0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, 0x63, 0x72, 0x61, 0x77, 0x6c, 0x12, 0x14,
+	0x0a, 0x05, 0x64, 0x65, 0x70, 0x74, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x05, 0x64,
+	0x65, 0x70, 0x74, 0x68, 0x12, 0x14, 0x0a, 0x05, 0x64, 0x65, 0x6c, 0x61, 0x79, 0x18, 0x04, 0x20,
+	0x01, 0x28, 0x03, 0x52, 0x05, 0x64, 0x65, 0x6c, 0x61, 0x79, 0x2a, 0xd9, 0x09, 0x0a, 0x0a, 0x53,
+	0x6f, 0x75, 0x72, 0x63, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x1d, 0x0a, 0x19, 0x53, 0x4f, 0x55,
+	0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x41, 0x5a, 0x55, 0x52, 0x45, 0x5f, 0x53,
+	0x54, 0x4f, 0x52, 0x41, 0x47, 0x45, 0x10, 0x00, 0x12, 0x19, 0x0a, 0x15, 0x53, 0x4f, 0x55, 0x52,
+	0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x42, 0x49, 0x54, 0x42, 0x55, 0x43, 0x4b, 0x45,
+	0x54, 0x10, 0x01, 0x12, 0x18, 0x0a, 0x14, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59,
+	0x50, 0x45, 0x5f, 0x43, 0x49, 0x52, 0x43, 0x4c, 0x45, 0x43, 0x49, 0x10, 0x02, 0x12, 0x1a, 0x0a,
+	0x16, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x43, 0x4f, 0x4e,
+	0x46, 0x4c, 0x55, 0x45, 0x4e, 0x43, 0x45, 0x10, 0x03, 0x12, 0x16, 0x0a, 0x12, 0x53, 0x4f, 0x55,
+	0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x44, 0x4f, 0x43, 0x4b, 0x45, 0x52, 0x10,
+	0x04, 0x12, 0x13, 0x0a, 0x0f, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45,
+	0x5f, 0x45, 0x43, 0x52, 0x10, 0x05, 0x12, 0x13, 0x0a, 0x0f, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45,
+	0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x43, 0x53, 0x10, 0x06, 0x12, 0x16, 0x0a, 0x12, 0x53,
+	0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x49, 0x54, 0x48, 0x55,
+	0x42, 0x10, 0x07, 0x12, 0x1a, 0x0a, 0x16, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59,
+	0x50, 0x45, 0x5f, 0x50, 0x55, 0x42, 0x4c, 0x49, 0x43, 0x5f, 0x47, 0x49, 0x54, 0x10, 0x08, 0x12,
+	0x16, 0x0a, 0x12, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47,
+	0x49, 0x54, 0x4c, 0x41, 0x42, 0x10, 0x09, 0x12, 0x14, 0x0a, 0x10, 0x53, 0x4f, 0x55, 0x52, 0x43,
+	0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4a, 0x49, 0x52, 0x41, 0x10, 0x0a, 0x12, 0x24, 0x0a,
+	0x20, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4e, 0x50, 0x4d,
+	0x5f, 0x55, 0x4e, 0x41, 0x55, 0x54, 0x48, 0x44, 0x5f, 0x50, 0x41, 0x43, 0x4b, 0x41, 0x47, 0x45,
+	0x53, 0x10, 0x0b, 0x12, 0x25, 0x0a, 0x21, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59,
+	0x50, 0x45, 0x5f, 0x50, 0x59, 0x50, 0x49, 0x5f, 0x55, 0x4e, 0x41, 0x55, 0x54, 0x48, 0x44, 0x5f,
+	0x50, 0x41, 0x43, 0x4b, 0x41, 0x47, 0x45, 0x53, 0x10, 0x0c, 0x12, 0x12, 0x0a, 0x0e, 0x53, 0x4f,
+	0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x33, 0x10, 0x0d, 0x12, 0x15,
+	0x0a, 0x11, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x4c,
+	0x41, 0x43, 0x4b, 0x10, 0x0e, 0x12, 0x1a, 0x0a, 0x16, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f,
+	0x54, 0x59, 0x50, 0x45, 0x5f, 0x46, 0x49, 0x4c, 0x45, 0x53, 0x59, 0x53, 0x54, 0x45, 0x4d, 0x10,
+	0x0f, 0x12, 0x13, 0x0a, 0x0f, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45,
+	0x5f, 0x47, 0x49, 0x54, 0x10, 0x10, 0x12, 0x14, 0x0a, 0x10, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45,
+	0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x54, 0x45, 0x53, 0x54, 0x10, 0x11, 0x12, 0x1b, 0x0a, 0x17,
+	0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x33, 0x5f, 0x55,
+	0x4e, 0x41, 0x55, 0x54, 0x48, 0x45, 0x44, 0x10, 0x12, 0x12, 0x2a, 0x0a, 0x26, 0x53, 0x4f, 0x55,
+	0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x49, 0x54, 0x48, 0x55, 0x42, 0x5f,
+	0x55, 0x4e, 0x41, 0x55, 0x54, 0x48, 0x45, 0x4e, 0x54, 0x49, 0x43, 0x41, 0x54, 0x45, 0x44, 0x5f,
+	0x4f, 0x52, 0x47, 0x10, 0x13, 0x12, 0x19, 0x0a, 0x15, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f,
+	0x54, 0x59, 0x50, 0x45, 0x5f, 0x42, 0x55, 0x49, 0x4c, 0x44, 0x4b, 0x49, 0x54, 0x45, 0x10, 0x14,
+	0x12, 0x16, 0x0a, 0x12, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f,
+	0x47, 0x45, 0x52, 0x52, 0x49, 0x54, 0x10, 0x15, 0x12, 0x17, 0x0a, 0x13, 0x53, 0x4f, 0x55, 0x52,
+	0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4a, 0x45, 0x4e, 0x4b, 0x49, 0x4e, 0x53, 0x10,
+	0x16, 0x12, 0x15, 0x0a, 0x11, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45,
+	0x5f, 0x54, 0x45, 0x41, 0x4d, 0x53, 0x10, 0x17, 0x12, 0x21, 0x0a, 0x1d, 0x53, 0x4f, 0x55, 0x52,
+	0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4a, 0x46, 0x52, 0x4f, 0x47, 0x5f, 0x41, 0x52,
+	0x54, 0x49, 0x46, 0x41, 0x43, 0x54, 0x4f, 0x52, 0x59, 0x10, 0x18, 0x12, 0x16, 0x0a, 0x12, 0x53,
+	0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x59, 0x53, 0x4c, 0x4f,
+	0x47, 0x10, 0x19, 0x12, 0x27, 0x0a, 0x23, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59,
+	0x50, 0x45, 0x5f, 0x50, 0x55, 0x42, 0x4c, 0x49, 0x43, 0x5f, 0x45, 0x56, 0x45, 0x4e, 0x54, 0x5f,
+	0x4d, 0x4f, 0x4e, 0x49, 0x54, 0x4f, 0x52, 0x49, 0x4e, 0x47, 0x10, 0x1a, 0x12, 0x1e, 0x0a, 0x1a,
+	0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x4c, 0x41, 0x43,
+	0x4b, 0x5f, 0x52, 0x45, 0x41, 0x4c, 0x54, 0x49, 0x4d, 0x45, 0x10, 0x1b, 0x12, 0x1c, 0x0a, 0x18,
+	0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x4f, 0x4f, 0x47,
+	0x4c, 0x45, 0x5f, 0x44, 0x52, 0x49, 0x56, 0x45, 0x10, 0x1c, 0x12, 0x1a, 0x0a, 0x16, 0x53, 0x4f,
+	0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x48, 0x41, 0x52, 0x45, 0x50,
+	0x4f, 0x49, 0x4e, 0x54, 0x10, 0x1d, 0x12, 0x1c, 0x0a, 0x18, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45,
+	0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x43, 0x53, 0x5f, 0x55, 0x4e, 0x41, 0x55, 0x54, 0x48,
+	0x45, 0x44, 0x10, 0x1e, 0x12, 0x1b, 0x0a, 0x17, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54,
+	0x59, 0x50, 0x45, 0x5f, 0x41, 0x5a, 0x55, 0x52, 0x45, 0x5f, 0x52, 0x45, 0x50, 0x4f, 0x53, 0x10,
+	0x1f, 0x12, 0x18, 0x0a, 0x14, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45,
+	0x5f, 0x54, 0x52, 0x41, 0x56, 0x49, 0x53, 0x43, 0x49, 0x10, 0x20, 0x12, 0x17, 0x0a, 0x13, 0x53,
+	0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x50, 0x4f, 0x53, 0x54, 0x4d,
+	0x41, 0x4e, 0x10, 0x21, 0x12, 0x17, 0x0a, 0x13, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54,
+	0x59, 0x50, 0x45, 0x5f, 0x57, 0x45, 0x42, 0x48, 0x4f, 0x4f, 0x4b, 0x10, 0x22, 0x12, 0x1d, 0x0a,
+	0x19, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x45, 0x4c, 0x41,
+	0x53, 0x54, 0x49, 0x43, 0x53, 0x45, 0x41, 0x52, 0x43, 0x48, 0x10, 0x23, 0x12, 0x1b, 0x0a, 0x17,
+	0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x48, 0x55, 0x47, 0x47,
+	0x49, 0x4e, 0x47, 0x46, 0x41, 0x43, 0x45, 0x10, 0x24, 0x12, 0x23, 0x0a, 0x1f, 0x53, 0x4f, 0x55,
 	0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x49, 0x54, 0x48, 0x55, 0x42, 0x5f,
-	0x52, 0x45, 0x41, 0x4c, 0x54, 0x49, 0x4d, 0x45, 0x10, 0x27, 0x12, 0x15, 0x0a, 0x11, 0x53, 0x4f,
-	0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x54, 0x44, 0x49, 0x4e, 0x10,
-	0x28, 0x12, 0x20, 0x0a, 0x1c, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45,
-	0x5f, 0x53, 0x4c, 0x41, 0x43, 0x4b, 0x5f, 0x43, 0x4f, 0x4e, 0x54, 0x49, 0x4e, 0x55, 0x4f, 0x55,
-	0x53, 0x10, 0x29, 0x12, 0x1f, 0x0a, 0x1b, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59,
-	0x50, 0x45, 0x5f, 0x4a, 0x53, 0x4f, 0x4e, 0x5f, 0x45, 0x4e, 0x55, 0x4d, 0x45, 0x52, 0x41, 0x54,
-	0x4f, 0x52, 0x10, 0x2a, 0x2a, 0x47, 0x0a, 0x19, 0x42, 0x69, 0x74, 0x62, 0x75, 0x63, 0x6b, 0x65,
-	0x74, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70,
-	0x65, 0x12, 0x0e, 0x0a, 0x0a, 0x41, 0x55, 0x54, 0x4f, 0x44, 0x45, 0x54, 0x45, 0x43, 0x54, 0x10,
-	0x00, 0x12, 0x09, 0x0a, 0x05, 0x43, 0x4c, 0x4f, 0x55, 0x44, 0x10, 0x01, 0x12, 0x0f, 0x0a, 0x0b,
-	0x44, 0x41, 0x54, 0x41, 0x5f, 0x43, 0x45, 0x4e, 0x54, 0x45, 0x52, 0x10, 0x02, 0x2a, 0x87, 0x01,
-	0x0a, 0x14, 0x4a, 0x69, 0x72, 0x61, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x61, 0x74, 0x69,
-	0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x12, 0x25, 0x0a, 0x21, 0x4a, 0x49, 0x52, 0x41, 0x5f, 0x49,
-	0x4e, 0x53, 0x54, 0x41, 0x4c, 0x4c, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x54, 0x59, 0x50, 0x45,
-	0x5f, 0x41, 0x55, 0x54, 0x4f, 0x44, 0x45, 0x54, 0x45, 0x43, 0x54, 0x10, 0x00, 0x12, 0x20, 0x0a,
-	0x1c, 0x4a, 0x49, 0x52, 0x41, 0x5f, 0x49, 0x4e, 0x53, 0x54, 0x41, 0x4c, 0x4c, 0x41, 0x54, 0x49,
-	0x4f, 0x4e, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x43, 0x4c, 0x4f, 0x55, 0x44, 0x10, 0x01, 0x12,
-	0x26, 0x0a, 0x22, 0x4a, 0x49, 0x52, 0x41, 0x5f, 0x49, 0x4e, 0x53, 0x54, 0x41, 0x4c, 0x4c, 0x41,
-	0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x44, 0x41, 0x54, 0x41, 0x5f, 0x43,
-	0x45, 0x4e, 0x54, 0x45, 0x52, 0x10, 0x02, 0x42, 0x3b, 0x5a, 0x39, 0x67, 0x69, 0x74, 0x68, 0x75,
-	0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73, 0x65, 0x63,
-	0x75, 0x72, 0x69, 0x74, 0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68, 0x6f, 0x67,
-	0x2f, 0x76, 0x33, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x73, 0x6f, 0x75, 0x72, 0x63,
-	0x65, 0x73, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
+	0x45, 0x58, 0x50, 0x45, 0x52, 0x49, 0x4d, 0x45, 0x4e, 0x54, 0x41, 0x4c, 0x10, 0x25, 0x12, 0x16,
+	0x0a, 0x12, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x45,
+	0x4e, 0x54, 0x52, 0x59, 0x10, 0x26, 0x12, 0x1f, 0x0a, 0x1b, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45,
+	0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x49, 0x54, 0x48, 0x55, 0x42, 0x5f, 0x52, 0x45, 0x41,
+	0x4c, 0x54, 0x49, 0x4d, 0x45, 0x10, 0x27, 0x12, 0x15, 0x0a, 0x11, 0x53, 0x4f, 0x55, 0x52, 0x43,
+	0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x54, 0x44, 0x49, 0x4e, 0x10, 0x28, 0x12, 0x20,
+	0x0a, 0x1c, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x4c,
+	0x41, 0x43, 0x4b, 0x5f, 0x43, 0x4f, 0x4e, 0x54, 0x49, 0x4e, 0x55, 0x4f, 0x55, 0x53, 0x10, 0x29,
+	0x12, 0x1f, 0x0a, 0x1b, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f,
+	0x4a, 0x53, 0x4f, 0x4e, 0x5f, 0x45, 0x4e, 0x55, 0x4d, 0x45, 0x52, 0x41, 0x54, 0x4f, 0x52, 0x10,
+	0x2a, 0x12, 0x13, 0x0a, 0x0f, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45,
+	0x5f, 0x57, 0x45, 0x42, 0x10, 0x2b, 0x2a, 0x47, 0x0a, 0x19, 0x42, 0x69, 0x74, 0x62, 0x75, 0x63,
+	0x6b, 0x65, 0x74, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x54,
+	0x79, 0x70, 0x65, 0x12, 0x0e, 0x0a, 0x0a, 0x41, 0x55, 0x54, 0x4f, 0x44, 0x45, 0x54, 0x45, 0x43,
+	0x54, 0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, 0x43, 0x4c, 0x4f, 0x55, 0x44, 0x10, 0x01, 0x12, 0x0f,
+	0x0a, 0x0b, 0x44, 0x41, 0x54, 0x41, 0x5f, 0x43, 0x45, 0x4e, 0x54, 0x45, 0x52, 0x10, 0x02, 0x2a,
+	0x87, 0x01, 0x0a, 0x14, 0x4a, 0x69, 0x72, 0x61, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x61,
+	0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x12, 0x25, 0x0a, 0x21, 0x4a, 0x49, 0x52, 0x41,
+	0x5f, 0x49, 0x4e, 0x53, 0x54, 0x41, 0x4c, 0x4c, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x54, 0x59,
+	0x50, 0x45, 0x5f, 0x41, 0x55, 0x54, 0x4f, 0x44, 0x45, 0x54, 0x45, 0x43, 0x54, 0x10, 0x00, 0x12,
+	0x20, 0x0a, 0x1c, 0x4a, 0x49, 0x52, 0x41, 0x5f, 0x49, 0x4e, 0x53, 0x54, 0x41, 0x4c, 0x4c, 0x41,
+	0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x43, 0x4c, 0x4f, 0x55, 0x44, 0x10,
+	0x01, 0x12, 0x26, 0x0a, 0x22, 0x4a, 0x49, 0x52, 0x41, 0x5f, 0x49, 0x4e, 0x53, 0x54, 0x41, 0x4c,
+	0x4c, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x44, 0x41, 0x54, 0x41,
+	0x5f, 0x43, 0x45, 0x4e, 0x54, 0x45, 0x52, 0x10, 0x02, 0x42, 0x3b, 0x5a, 0x39, 0x67, 0x69, 0x74,
+	0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73,
+	0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68,
+	0x6f, 0x67, 0x2f, 0x76, 0x33, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x73, 0x6f, 0x75,
+	0x72, 0x63, 0x65, 0x73, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
 }
 
 var (
@@ -6002,7 +6083,7 @@ func file_sources_proto_rawDescGZIP() []byte {
 }
 
 var file_sources_proto_enumTypes = make([]protoimpl.EnumInfo, 4)
-var file_sources_proto_msgTypes = make([]protoimpl.MessageInfo, 41)
+var file_sources_proto_msgTypes = make([]protoimpl.MessageInfo, 42)
 var file_sources_proto_goTypes = []interface{}{
 	(SourceType)(0),                             // 0: sources.SourceType
 	(BitbucketInstallationType)(0),              // 1: sources.BitbucketInstallationType
@@ -6049,80 +6130,81 @@ var file_sources_proto_goTypes = []interface{}{
 	(*Stdin)(nil),                               // 42: sources.Stdin
 	(*SlackContinuous)(nil),                     // 43: sources.SlackContinuous
 	(*JSONEnumerator)(nil),                      // 44: sources.JSONEnumerator
-	(*durationpb.Duration)(nil),                 // 45: google.protobuf.Duration
-	(*anypb.Any)(nil),                           // 46: google.protobuf.Any
-	(*credentialspb.BasicAuth)(nil),             // 47: credentials.BasicAuth
-	(*credentialspb.Unauthenticated)(nil),       // 48: credentials.Unauthenticated
-	(*credentialspb.Oauth2)(nil),                // 49: credentials.Oauth2
-	(*credentialspb.KeySecret)(nil),             // 50: credentials.KeySecret
-	(*credentialspb.CloudEnvironment)(nil),      // 51: credentials.CloudEnvironment
-	(*credentialspb.SSHAuth)(nil),               // 52: credentials.SSHAuth
-	(*credentialspb.GitHubApp)(nil),             // 53: credentials.GitHubApp
-	(*credentialspb.GoogleDriveDWD)(nil),        // 54: credentials.GoogleDriveDWD
-	(*credentialspb.AWSSessionTokenSecret)(nil), // 55: credentials.AWSSessionTokenSecret
-	(*credentialspb.SlackTokens)(nil),           // 56: credentials.SlackTokens
-	(*credentialspb.Header)(nil),                // 57: credentials.Header
-	(*credentialspb.ClientCredentials)(nil),     // 58: credentials.ClientCredentials
-	(*timestamppb.Timestamp)(nil),               // 59: google.protobuf.Timestamp
+	(*Web)(nil),                                 // 45: sources.Web
+	(*durationpb.Duration)(nil),                 // 46: google.protobuf.Duration
+	(*anypb.Any)(nil),                           // 47: google.protobuf.Any
+	(*credentialspb.BasicAuth)(nil),             // 48: credentials.BasicAuth
+	(*credentialspb.Unauthenticated)(nil),       // 49: credentials.Unauthenticated
+	(*credentialspb.Oauth2)(nil),                // 50: credentials.Oauth2
+	(*credentialspb.KeySecret)(nil),             // 51: credentials.KeySecret
+	(*credentialspb.CloudEnvironment)(nil),      // 52: credentials.CloudEnvironment
+	(*credentialspb.SSHAuth)(nil),               // 53: credentials.SSHAuth
+	(*credentialspb.GitHubApp)(nil),             // 54: credentials.GitHubApp
+	(*credentialspb.GoogleDriveDWD)(nil),        // 55: credentials.GoogleDriveDWD
+	(*credentialspb.AWSSessionTokenSecret)(nil), // 56: credentials.AWSSessionTokenSecret
+	(*credentialspb.SlackTokens)(nil),           // 57: credentials.SlackTokens
+	(*credentialspb.Header)(nil),                // 58: credentials.Header
+	(*credentialspb.ClientCredentials)(nil),     // 59: credentials.ClientCredentials
+	(*timestamppb.Timestamp)(nil),               // 60: google.protobuf.Timestamp
 }
 var file_sources_proto_depIdxs = []int32{
-	45, // 0: sources.LocalSource.scan_interval:type_name -> google.protobuf.Duration
-	46, // 1: sources.LocalSource.connection:type_name -> google.protobuf.Any
-	47, // 2: sources.Artifactory.basic_auth:type_name -> credentials.BasicAuth
-	48, // 3: sources.Artifactory.unauthenticated:type_name -> credentials.Unauthenticated
-	47, // 4: sources.AzureStorage.basic_auth:type_name -> credentials.BasicAuth
-	48, // 5: sources.AzureStorage.unauthenticated:type_name -> credentials.Unauthenticated
-	49, // 6: sources.Bitbucket.oauth:type_name -> credentials.Oauth2
-	47, // 7: sources.Bitbucket.basic_auth:type_name -> credentials.BasicAuth
+	46, // 0: sources.LocalSource.scan_interval:type_name -> google.protobuf.Duration
+	47, // 1: sources.LocalSource.connection:type_name -> google.protobuf.Any
+	48, // 2: sources.Artifactory.basic_auth:type_name -> credentials.BasicAuth
+	49, // 3: sources.Artifactory.unauthenticated:type_name -> credentials.Unauthenticated
+	48, // 4: sources.AzureStorage.basic_auth:type_name -> credentials.BasicAuth
+	49, // 5: sources.AzureStorage.unauthenticated:type_name -> credentials.Unauthenticated
+	50, // 6: sources.Bitbucket.oauth:type_name -> credentials.Oauth2
+	48, // 7: sources.Bitbucket.basic_auth:type_name -> credentials.BasicAuth
 	1,  // 8: sources.Bitbucket.installation_type:type_name -> sources.BitbucketInstallationType
-	48, // 9: sources.Confluence.unauthenticated:type_name -> credentials.Unauthenticated
-	47, // 10: sources.Confluence.basic_auth:type_name -> credentials.BasicAuth
+	49, // 9: sources.Confluence.unauthenticated:type_name -> credentials.Unauthenticated
+	48, // 10: sources.Confluence.basic_auth:type_name -> credentials.BasicAuth
 	3,  // 11: sources.Confluence.spaces_scope:type_name -> sources.Confluence.GetAllSpacesScope
-	48, // 12: sources.Docker.unauthenticated:type_name -> credentials.Unauthenticated
-	47, // 13: sources.Docker.basic_auth:type_name -> credentials.BasicAuth
-	50, // 14: sources.ECR.access_key:type_name -> credentials.KeySecret
-	48, // 15: sources.GCS.unauthenticated:type_name -> credentials.Unauthenticated
-	51, // 16: sources.GCS.adc:type_name -> credentials.CloudEnvironment
-	49, // 17: sources.GCS.oauth:type_name -> credentials.Oauth2
-	47, // 18: sources.Git.basic_auth:type_name -> credentials.BasicAuth
-	48, // 19: sources.Git.unauthenticated:type_name -> credentials.Unauthenticated
-	52, // 20: sources.Git.ssh_auth:type_name -> credentials.SSHAuth
-	49, // 21: sources.GitLab.oauth:type_name -> credentials.Oauth2
-	47, // 22: sources.GitLab.basic_auth:type_name -> credentials.BasicAuth
-	53, // 23: sources.GitHub.github_app:type_name -> credentials.GitHubApp
-	48, // 24: sources.GitHub.unauthenticated:type_name -> credentials.Unauthenticated
-	47, // 25: sources.GitHub.basic_auth:type_name -> credentials.BasicAuth
-	53, // 26: sources.GitHubRealtime.github_app:type_name -> credentials.GitHubApp
-	48, // 27: sources.GitHubRealtime.unauthenticated:type_name -> credentials.Unauthenticated
-	47, // 28: sources.GitHubRealtime.basic_auth:type_name -> credentials.BasicAuth
-	49, // 29: sources.GoogleDrive.oauth:type_name -> credentials.Oauth2
-	54, // 30: sources.GoogleDrive.dwd:type_name -> credentials.GoogleDriveDWD
-	48, // 31: sources.Huggingface.unauthenticated:type_name -> credentials.Unauthenticated
-	47, // 32: sources.JIRA.basic_auth:type_name -> credentials.BasicAuth
-	48, // 33: sources.JIRA.unauthenticated:type_name -> credentials.Unauthenticated
-	49, // 34: sources.JIRA.oauth:type_name -> credentials.Oauth2
+	49, // 12: sources.Docker.unauthenticated:type_name -> credentials.Unauthenticated
+	48, // 13: sources.Docker.basic_auth:type_name -> credentials.BasicAuth
+	51, // 14: sources.ECR.access_key:type_name -> credentials.KeySecret
+	49, // 15: sources.GCS.unauthenticated:type_name -> credentials.Unauthenticated
+	52, // 16: sources.GCS.adc:type_name -> credentials.CloudEnvironment
+	50, // 17: sources.GCS.oauth:type_name -> credentials.Oauth2
+	48, // 18: sources.Git.basic_auth:type_name -> credentials.BasicAuth
+	49, // 19: sources.Git.unauthenticated:type_name -> credentials.Unauthenticated
+	53, // 20: sources.Git.ssh_auth:type_name -> credentials.SSHAuth
+	50, // 21: sources.GitLab.oauth:type_name -> credentials.Oauth2
+	48, // 22: sources.GitLab.basic_auth:type_name -> credentials.BasicAuth
+	54, // 23: sources.GitHub.github_app:type_name -> credentials.GitHubApp
+	49, // 24: sources.GitHub.unauthenticated:type_name -> credentials.Unauthenticated
+	48, // 25: sources.GitHub.basic_auth:type_name -> credentials.BasicAuth
+	54, // 26: sources.GitHubRealtime.github_app:type_name -> credentials.GitHubApp
+	49, // 27: sources.GitHubRealtime.unauthenticated:type_name -> credentials.Unauthenticated
+	48, // 28: sources.GitHubRealtime.basic_auth:type_name -> credentials.BasicAuth
+	50, // 29: sources.GoogleDrive.oauth:type_name -> credentials.Oauth2
+	55, // 30: sources.GoogleDrive.dwd:type_name -> credentials.GoogleDriveDWD
+	49, // 31: sources.Huggingface.unauthenticated:type_name -> credentials.Unauthenticated
+	48, // 32: sources.JIRA.basic_auth:type_name -> credentials.BasicAuth
+	49, // 33: sources.JIRA.unauthenticated:type_name -> credentials.Unauthenticated
+	50, // 34: sources.JIRA.oauth:type_name -> credentials.Oauth2
 	2,  // 35: sources.JIRA.installation_type:type_name -> sources.JiraInstallationType
-	48, // 36: sources.NPMUnauthenticatedPackage.unauthenticated:type_name -> credentials.Unauthenticated
-	48, // 37: sources.PyPIUnauthenticatedPackage.unauthenticated:type_name -> credentials.Unauthenticated
-	50, // 38: sources.S3.access_key:type_name -> credentials.KeySecret
-	48, // 39: sources.S3.unauthenticated:type_name -> credentials.Unauthenticated
-	51, // 40: sources.S3.cloud_environment:type_name -> credentials.CloudEnvironment
-	55, // 41: sources.S3.session_token:type_name -> credentials.AWSSessionTokenSecret
-	56, // 42: sources.Slack.tokens:type_name -> credentials.SlackTokens
-	47, // 43: sources.Gerrit.basic_auth:type_name -> credentials.BasicAuth
-	48, // 44: sources.Gerrit.unauthenticated:type_name -> credentials.Unauthenticated
-	47, // 45: sources.Jenkins.basic_auth:type_name -> credentials.BasicAuth
-	57, // 46: sources.Jenkins.header:type_name -> credentials.Header
-	48, // 47: sources.Jenkins.unauthenticated:type_name -> credentials.Unauthenticated
-	58, // 48: sources.Teams.authenticated:type_name -> credentials.ClientCredentials
-	49, // 49: sources.Teams.oauth:type_name -> credentials.Oauth2
-	48, // 50: sources.Forager.unauthenticated:type_name -> credentials.Unauthenticated
-	59, // 51: sources.Forager.since:type_name -> google.protobuf.Timestamp
-	56, // 52: sources.SlackRealtime.tokens:type_name -> credentials.SlackTokens
-	49, // 53: sources.Sharepoint.oauth:type_name -> credentials.Oauth2
-	49, // 54: sources.AzureRepos.oauth:type_name -> credentials.Oauth2
-	48, // 55: sources.Postman.unauthenticated:type_name -> credentials.Unauthenticated
-	57, // 56: sources.Webhook.header:type_name -> credentials.Header
+	49, // 36: sources.NPMUnauthenticatedPackage.unauthenticated:type_name -> credentials.Unauthenticated
+	49, // 37: sources.PyPIUnauthenticatedPackage.unauthenticated:type_name -> credentials.Unauthenticated
+	51, // 38: sources.S3.access_key:type_name -> credentials.KeySecret
+	49, // 39: sources.S3.unauthenticated:type_name -> credentials.Unauthenticated
+	52, // 40: sources.S3.cloud_environment:type_name -> credentials.CloudEnvironment
+	56, // 41: sources.S3.session_token:type_name -> credentials.AWSSessionTokenSecret
+	57, // 42: sources.Slack.tokens:type_name -> credentials.SlackTokens
+	48, // 43: sources.Gerrit.basic_auth:type_name -> credentials.BasicAuth
+	49, // 44: sources.Gerrit.unauthenticated:type_name -> credentials.Unauthenticated
+	48, // 45: sources.Jenkins.basic_auth:type_name -> credentials.BasicAuth
+	58, // 46: sources.Jenkins.header:type_name -> credentials.Header
+	49, // 47: sources.Jenkins.unauthenticated:type_name -> credentials.Unauthenticated
+	59, // 48: sources.Teams.authenticated:type_name -> credentials.ClientCredentials
+	50, // 49: sources.Teams.oauth:type_name -> credentials.Oauth2
+	49, // 50: sources.Forager.unauthenticated:type_name -> credentials.Unauthenticated
+	60, // 51: sources.Forager.since:type_name -> google.protobuf.Timestamp
+	57, // 52: sources.SlackRealtime.tokens:type_name -> credentials.SlackTokens
+	50, // 53: sources.Sharepoint.oauth:type_name -> credentials.Oauth2
+	50, // 54: sources.AzureRepos.oauth:type_name -> credentials.Oauth2
+	49, // 55: sources.Postman.unauthenticated:type_name -> credentials.Unauthenticated
+	58, // 56: sources.Webhook.header:type_name -> credentials.Header
 	39, // 57: sources.Webhook.vector:type_name -> sources.Vector
 	58, // [58:58] is the sub-list for method output_type
 	58, // [58:58] is the sub-list for method input_type
@@ -6629,6 +6711,18 @@ func file_sources_proto_init() {
 				return nil
 			}
 		}
+		file_sources_proto_msgTypes[41].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*Web); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
 	}
 	file_sources_proto_msgTypes[1].OneofWrappers = []interface{}{
 		(*Artifactory_BasicAuth)(nil),
@@ -6780,7 +6874,7 @@ func file_sources_proto_init() {
 			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
 			RawDescriptor: file_sources_proto_rawDesc,
 			NumEnums:      4,
-			NumMessages:   41,
+			NumMessages:   42,
 			NumExtensions: 0,
 			NumServices:   0,
 		},
diff --git a/pkg/pb/sourcespb/sources.pb.validate.go b/pkg/pb/sourcespb/sources.pb.validate.go
index eca7b0d6a469..d64382b13e87 100644
--- a/pkg/pb/sourcespb/sources.pb.validate.go
+++ b/pkg/pb/sourcespb/sources.pb.validate.go
@@ -7238,3 +7238,107 @@ var _ interface {
 	Cause() error
 	ErrorName() string
 } = JSONEnumeratorValidationError{}
+
+// Validate checks the field values on Web with the rules defined in the proto
+// definition for this message. If any rules are violated, the first error
+// encountered is returned, or nil if there are no violations.
+func (m *Web) Validate() error {
+	return m.validate(false)
+}
+
+// ValidateAll checks the field values on Web with the rules defined in the
+// proto definition for this message. If any rules are violated, the result is
+// a list of violation errors wrapped in WebMultiError, or nil if none found.
+func (m *Web) ValidateAll() error {
+	return m.validate(true)
+}
+
+func (m *Web) validate(all bool) error {
+	if m == nil {
+		return nil
+	}
+
+	var errors []error
+
+	// no validation rules for Crawl
+
+	// no validation rules for Depth
+
+	// no validation rules for Delay
+
+	if len(errors) > 0 {
+		return WebMultiError(errors)
+	}
+
+	return nil
+}
+
+// WebMultiError is an error wrapping multiple validation errors returned by
+// Web.ValidateAll() if the designated constraints aren't met.
+type WebMultiError []error
+
+// Error returns a concatenation of all the error messages it wraps.
+func (m WebMultiError) Error() string {
+	var msgs []string
+	for _, err := range m {
+		msgs = append(msgs, err.Error())
+	}
+	return strings.Join(msgs, "; ")
+}
+
+// AllErrors returns a list of validation violation errors.
+func (m WebMultiError) AllErrors() []error { return m }
+
+// WebValidationError is the validation error returned by Web.Validate if the
+// designated constraints aren't met.
+type WebValidationError struct {
+	field  string
+	reason string
+	cause  error
+	key    bool
+}
+
+// Field function returns field value.
+func (e WebValidationError) Field() string { return e.field }
+
+// Reason function returns reason value.
+func (e WebValidationError) Reason() string { return e.reason }
+
+// Cause function returns cause value.
+func (e WebValidationError) Cause() error { return e.cause }
+
+// Key function returns key value.
+func (e WebValidationError) Key() bool { return e.key }
+
+// ErrorName returns error name.
+func (e WebValidationError) ErrorName() string { return "WebValidationError" }
+
+// Error satisfies the builtin error interface
+func (e WebValidationError) Error() string {
+	cause := ""
+	if e.cause != nil {
+		cause = fmt.Sprintf(" | caused by: %v", e.cause)
+	}
+
+	key := ""
+	if e.key {
+		key = "key for "
+	}
+
+	return fmt.Sprintf(
+		"invalid %sWeb.%s: %s%s",
+		key,
+		e.field,
+		e.reason,
+		cause)
+}
+
+var _ error = WebValidationError{}
+
+var _ interface {
+	Field() string
+	Reason() string
+	Key() bool
+	Cause() error
+	ErrorName() string
+} = WebValidationError{}
diff --git a/pkg/sources/sources.go b/pkg/sources/sources.go
index 88b11b85a5ea..163c8681390e 100644
--- a/pkg/sources/sources.go
+++ b/pkg/sources/sources.go
@@ -482,6 +482,22 @@ type JSONEnumeratorConfig struct {
 	Paths []string
 }
 
+// WebConfig defines the configuration for the web source.
+type WebConfig struct {
+	// URL is the list of starting points for scanning.
+	URLs []string `json:"url" mapstructure:"url"`
+
+	// Crawl determines whether to follow links from the starting page.
+	Crawl bool `json:"crawl" mapstructure:"crawl"`
+
+	// Depth controls how many link hops to follow when Crawl is true.
+	// 0 = only the starting URL, 1 = starting URL and direct links, etc.
+	Depth int `json:"depth" mapstructure:"depth"`
+
+	// Delay is the delay (in seconds) between requests to the same domain.
+	Delay int `json:"delay" mapstructure:"delay"`
+}
+
 // Progress is used to update job completion progress across sources.
 type Progress struct {
 	mut sync.Mutex
diff --git a/pkg/sources/web/web.go b/pkg/sources/web/web.go
new file mode 100644
index 000000000000..adc75af386d4
--- /dev/null
+++ b/pkg/sources/web/web.go
@@ -0,0 +1,42 @@
+package web
+
+import (
+	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
+	"google.golang.org/protobuf/types/known/anypb"
+)
+
+const SourceType = sourcespb.SourceType_SOURCE_TYPE_WEB
+
+type Source struct {
+	name        string
+	sourceId    sources.SourceID
+	jobId       sources.JobID
+	verify      bool
+	concurrency int
+	conn        sourcespb.Web
+
+	sources.Progress
+	sources.CommonSourceUnitUnmarshaller
+}
+
+// Ensure the Source satisfies the interfaces at compile time
+var _ sources.Source = (*Source)(nil)
+var _ sources.SourceUnitUnmarshaller = (*Source)(nil)
+
+func (s *Source) Type() sourcespb.SourceType { return SourceType }
+func (s *Source) SourceID() sources.SourceID { return s.sourceId }
+func (s *Source) JobID() sources.JobID       { return s.jobId }
+
+// Init initializes the source.
+func (s *Source) Init(ctx context.Context, name string, jobId sources.JobID, sourceId sources.SourceID,
+	verify bool, connection *anypb.Any, concurrency int,
+) error {
+	return nil
+}
+
+// Chunks emits data over a channel that is decoded and scanned for secrets.
+func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ ...sources.ChunkingTarget) error {
+	return nil
+}
diff --git a/proto/sources.proto b/proto/sources.proto
index cf64caca8687..8b94a629a572 100644
--- a/proto/sources.proto
+++ b/proto/sources.proto
@@ -55,6 +55,7 @@ enum SourceType {
   SOURCE_TYPE_STDIN = 40;
   SOURCE_TYPE_SLACK_CONTINUOUS = 41;
   SOURCE_TYPE_JSON_ENUMERATOR = 42;
+  SOURCE_TYPE_WEB = 43;
 }
 
 message LocalSource {
@@ -554,3 +555,10 @@ message SlackContinuous {
 message JSONEnumerator {
   repeated string paths = 1;
 }
+
+message Web {
+  repeated string urls = 1;
+  bool crawl = 2;
+  int64 depth = 3;
+  int64 delay = 4;
+}

From 9e3fbe5e1cf1956ab06efaf9ff536b287639a468 Mon Sep 17 00:00:00 2001
From: Kashif Khan <kashif.khan@trufflesec.com>
Date: Fri, 27 Mar 2026 17:38:10 +0500
Subject: [PATCH 02/20] it works end to end

---
 go.mod                                        |  15 +-
 go.sum                                        |  74 ++++++
 main.go                                       |  23 ++
 pkg/engine/web.go                             |  39 +++
 .../source_metadatapb/source_metadata.pb.go   | 227 +++++++++++++-----
 .../source_metadata.pb.validate.go            | 149 ++++++++++++
 pkg/sources/web/web.go                        | 171 ++++++++++++-
 proto/source_metadata.proto                   |   9 +
 8 files changed, 651 insertions(+), 56 deletions(-)
 create mode 100644 pkg/engine/web.go

diff --git a/go.mod b/go.mod
index e075fea6105e..f063588c7388 100644
--- a/go.mod
+++ b/go.mod
@@ -138,10 +138,15 @@ require (
 	github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.53.0 // indirect
 	github.com/Microsoft/go-winio v0.6.2 // indirect
 	github.com/ProtonMail/go-crypto v1.1.5 // indirect
+	github.com/PuerkitoBio/goquery v1.11.0 // indirect
 	github.com/STARRY-S/zip v0.2.1 // indirect
 	github.com/alecthomas/chroma/v2 v2.14.0 // indirect
 	github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect
 	github.com/andybalholm/brotli v1.1.1 // indirect
+	github.com/andybalholm/cascadia v1.3.3 // indirect
+	github.com/antchfx/htmlquery v1.3.5 // indirect
+	github.com/antchfx/xmlquery v1.5.0 // indirect
+	github.com/antchfx/xpath v1.3.5 // indirect
 	github.com/atotto/clipboard v0.1.4 // indirect
 	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.1 // indirect
 	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.7 // indirect
@@ -158,6 +163,7 @@ require (
 	github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
 	github.com/aymerick/douceur v0.2.0 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/bits-and-blooms/bitset v1.24.4 // indirect
 	github.com/bodgit/plumbing v1.3.0 // indirect
 	github.com/bodgit/sevenzip v1.6.0 // indirect
 	github.com/bodgit/windows v1.0.1 // indirect
@@ -204,12 +210,14 @@ require (
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-ole/go-ole v1.2.6 // indirect
 	github.com/goccy/go-json v0.10.3 // indirect
+	github.com/gocolly/colly/v2 v2.3.0 // indirect
 	github.com/gofrs/flock v0.12.1 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang-jwt/jwt/v4 v4.5.2 // indirect
 	github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 // indirect
 	github.com/golang-sql/sqlexp v0.1.0 // indirect
-	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
+	github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect
+	github.com/golang/protobuf v1.5.4 // indirect
 	github.com/golang/snappy v1.0.0 // indirect
 	github.com/google/go-github/v72 v72.0.0 // indirect
 	github.com/google/go-querystring v1.2.0 // indirect
@@ -226,6 +234,7 @@ require (
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
 	github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
 	github.com/jpillora/s3 v1.1.4 // indirect
+	github.com/kennygrant/sanitize v1.2.4 // indirect
 	github.com/kevinburke/ssh_config v1.2.0 // indirect
 	github.com/kjk/lzma v0.0.0-20161016003348-3fd93898850d // indirect
 	github.com/klauspost/compress v1.18.0 // indirect
@@ -256,6 +265,7 @@ require (
 	github.com/muesli/cancelreader v0.2.2 // indirect
 	github.com/muesli/termenv v0.16.0 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/nlnwa/whatwg-url v0.6.2 // indirect
 	github.com/nwaples/rardecode/v2 v2.2.1 // indirect
 	github.com/onsi/ginkgo v1.16.5 // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
@@ -270,6 +280,7 @@ require (
 	github.com/prometheus/procfs v0.15.1 // indirect
 	github.com/rivo/uniseg v0.4.7 // indirect
 	github.com/sahilm/fuzzy v0.1.1-0.20230530133925-c48e322e2a8f // indirect
+	github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect
 	github.com/segmentio/asm v1.2.1 // indirect
 	github.com/sendgrid/rest v2.6.9+incompatible // indirect
 	github.com/shirou/gopsutil/v3 v3.23.12 // indirect
@@ -280,6 +291,7 @@ require (
 	github.com/sorairolake/lzip-go v0.3.5 // indirect
 	github.com/spiffe/go-spiffe/v2 v2.6.0 // indirect
 	github.com/stretchr/objx v0.5.2 // indirect
+	github.com/temoto/robotstxt v1.1.2 // indirect
 	github.com/tetratelabs/wazero v1.9.0 // indirect
 	github.com/therootcompany/xz v1.0.1 // indirect
 	github.com/tklauser/go-sysconf v0.3.12 // indirect
@@ -314,6 +326,7 @@ require (
 	golang.org/x/mod v0.30.0 // indirect
 	golang.org/x/sys v0.39.0 // indirect
 	golang.org/x/term v0.38.0 // indirect
+	google.golang.org/appengine v1.6.8 // indirect
 	google.golang.org/genproto v0.0.0-20251202230838-ff82c1b0f217 // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20251222181119-0a764e51fe1b // indirect
diff --git a/go.sum b/go.sum
index 540dbc6bacb3..9c6307c7e797 100644
--- a/go.sum
+++ b/go.sum
@@ -80,6 +80,8 @@ github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERo
 github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
 github.com/ProtonMail/go-crypto v1.1.5 h1:eoAQfK2dwL+tFSFpr7TbOaPNUbPiJj4fLYwwGE1FQO4=
 github.com/ProtonMail/go-crypto v1.1.5/go.mod h1:rA3QumHc/FZ8pAHreoekgiAbzpNsfQAosU5td4SnOrE=
+github.com/PuerkitoBio/goquery v1.11.0 h1:jZ7pwMQXIITcUXNH83LLk+txlaEy6NVOfTuP43xxfqw=
+github.com/PuerkitoBio/goquery v1.11.0/go.mod h1:wQHgxUOU3JGuj3oD/QFfxUdlzW6xPHfqyHre6VMY4DQ=
 github.com/STARRY-S/zip v0.2.1 h1:pWBd4tuSGm3wtpoqRZZ2EAwOmcHK6XFf7bU9qcJXyFg=
 github.com/STARRY-S/zip v0.2.1/go.mod h1:xNvshLODWtC4EJ702g7cTYn13G53o1+X9BWnPFpcWV4=
 github.com/TheZeroSlave/zapsentry v1.23.0 h1:TKyzfEL7LRlRr+7AvkukVLZ+jZPC++ebCUv7ZJHl1AU=
@@ -98,8 +100,16 @@ github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAu
 github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE=
 github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA=
 github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA=
+github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kktS1LM=
+github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA=
 github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8=
 github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4=
+github.com/antchfx/htmlquery v1.3.5 h1:aYthDDClnG2a2xePf6tys/UyyM/kRcsFRm+ifhFKoU0=
+github.com/antchfx/htmlquery v1.3.5/go.mod h1:5oyIPIa3ovYGtLqMPNjBF2Uf25NPCKsMjCnQ8lvjaoA=
+github.com/antchfx/xmlquery v1.5.0 h1:uAi+mO40ZWfyU6mlUBxRVvL6uBNZ6LMU4M3+mQIBV4c=
+github.com/antchfx/xmlquery v1.5.0/go.mod h1:lJfWRXzYMK1ss32zm1GQV3gMIW/HFey3xDZmkP1SuNc=
+github.com/antchfx/xpath v1.3.5 h1:PqbXLC3TkfeZyakF5eeh3NTWEbYl4VHNVeufANzDbKQ=
+github.com/antchfx/xpath v1.3.5/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs=
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
 github.com/atotto/clipboard v0.1.4 h1:EH0zSVneZPSuFR11BlR9YppQTVDbh5+16AmcJi4g1z4=
@@ -159,6 +169,9 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/bill-rich/go-syslog v0.0.0-20220413021637-49edb52a574c h1:tSME5FDS02qQll3JYodI6RZR/g4EKOHApGv1wMZT+Z0=
 github.com/bill-rich/go-syslog v0.0.0-20220413021637-49edb52a574c/go.mod h1:+sCc6hztur+oZCLOsNk6wCCy+GLrnSNHSRmTnnL+8iQ=
+github.com/bits-and-blooms/bitset v1.20.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8=
+github.com/bits-and-blooms/bitset v1.24.4 h1:95H15Og1clikBrKr/DuzMXkQzECs1M6hhoGXLwLQOZE=
+github.com/bits-and-blooms/bitset v1.24.4/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8=
 github.com/bodgit/plumbing v1.3.0 h1:pf9Itz1JOQgn7vEOE7v7nlEfBykYqvUYioC61TwWCFU=
 github.com/bodgit/plumbing v1.3.0/go.mod h1:JOTb4XiRu5xfnmdnDJo6GmSbSbtSyufrsyZFByMtKEs=
 github.com/bodgit/sevenzip v1.6.0 h1:a4R0Wu6/P1o1pP/3VV++aEOcyeBxeO/xE2Y9NSTrr6A=
@@ -348,6 +361,10 @@ github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6Wezm
 github.com/gobwas/ws v1.2.1/go.mod h1:hRKAFb8wOxFROYNsT1bqfWnhX+b5MFeJM9r2ZSwg/KY=
 github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA=
 github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
+github.com/gocolly/colly v1.2.0 h1:qRz9YAn8FIH0qzgNUw+HT9UN7wm1oF9OBAilwEWpyrI=
+github.com/gocolly/colly v1.2.0/go.mod h1:Hof5T3ZswNVsOHYmba1u03W65HDWgpV5HifSuueE0EA=
+github.com/gocolly/colly/v2 v2.3.0 h1:HSFh0ckbgVd2CSGRE+Y/iA4goUhGROJwyQDCMXGFBWM=
+github.com/gocolly/colly/v2 v2.3.0/go.mod h1:Qp54s/kQbwCQvFVx8KzKCSTXVJ1wWT4QeAKEu33x1q8=
 github.com/gofrs/flock v0.12.1 h1:MTLVXXHf8ekldpJk3AKicLij9MdwOWkZ+a/jHHZby9E=
 github.com/gofrs/flock v0.12.1/go.mod h1:9zxTsyu5xtJ9DK+1tFZyibEV7y3uwDxPPfbxeeHCoD0=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
@@ -366,6 +383,8 @@ github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4er
 github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE=
 github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 h1:f+oWsMOmNPc8JmEHVZIycC7hBoQxHH9pNKQORJNozsQ=
+github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8/go.mod h1:wcDNUvekVysuuOpQKo3191zZyTpiI6se1N1ULghS0sw=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
@@ -380,6 +399,8 @@ github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrU
 github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
 github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
 github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
+github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
 github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
 github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
 github.com/golang/snappy v1.0.0 h1:Oy607GVXHs7RtbggtPBnr2RmDArIsAefDwvrdWvRhGs=
@@ -478,6 +499,8 @@ github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1
 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
 github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213 h1:qGQQKEcAR99REcMpsXCp3lJ03zYT1PkRd3kQGPn9GVg=
 github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213/go.mod h1:vNUNkEQ1e29fT/6vq2aBdFsgNPmy8qMdSay1npru+Sw=
+github.com/kennygrant/sanitize v1.2.4 h1:gN25/otpP5vAsO2djbMhF/LQX6R7+O1TB4yv8NzpJ3o=
+github.com/kennygrant/sanitize v1.2.4/go.mod h1:LGsjYYtgxbetdg5owWB2mpgUL6e2nfw2eObZ0u0qvak=
 github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4=
 github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
@@ -587,6 +610,8 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8m
 github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32 h1:W6apQkHrMkS0Muv8G/TipAy/FJl/rCYT0+EuS8+Z0z4=
 github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32/go.mod h1:9wM+0iRr9ahx58uYLpLIr5fm8diHn0JbqRycJi6w0Ms=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
+github.com/nlnwa/whatwg-url v0.6.2 h1:jU61lU2ig4LANydbEJmA2nPrtCGiKdtgT0rmMd2VZ/Q=
+github.com/nlnwa/whatwg-url v0.6.2/go.mod h1:x0FPXJzzOEieQtsBT/AKvbiBbQ46YlL6Xa7m02M1ECk=
 github.com/nwaples/rardecode/v2 v2.2.1 h1:DgHK/O/fkTQEKBJxBMC5d9IU8IgauifbpG78+rZJMnI=
 github.com/nwaples/rardecode/v2 v2.2.1/go.mod h1:7uz379lSxPe6j9nvzxUZ+n7mnJNgjsRNb6IbvGVHRmw=
 github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
@@ -653,6 +678,8 @@ github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7
 github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk=
 github.com/sahilm/fuzzy v0.1.1-0.20230530133925-c48e322e2a8f h1:MvTmaQdww/z0Q4wrYjDSCcZ78NoftLQyHBSLW/Cx79Y=
 github.com/sahilm/fuzzy v0.1.1-0.20230530133925-c48e322e2a8f/go.mod h1:VFvziUEIMCrT6A6tw2RFIXPXXmzXbOsSHF0DOI8ZK9Y=
+github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d h1:hrujxIzL1woJ7AwssoOcM/tq5JjjG2yYOc8odClEiXA=
+github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU=
 github.com/sassoftware/go-rpmutils v0.4.0 h1:ojND82NYBxgwrV+mX1CWsd5QJvvEZTKddtCdFLPWhpg=
 github.com/sassoftware/go-rpmutils v0.4.0/go.mod h1:3goNWi7PGAT3/dlql2lv3+MSN5jNYPjT5mVcQcIsYzI=
 github.com/schollz/progressbar/v3 v3.17.1 h1:bI1MTaoQO+v5kzklBjYNRQLoVpe0zbyRZNK6DFkVC5U=
@@ -709,6 +736,8 @@ github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o
 github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
 github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
 github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
+github.com/temoto/robotstxt v1.1.2 h1:W2pOjSJ6SWvldyEuiFXNxz3xZ8aiWX5LbfDiOFd7Fxg=
+github.com/temoto/robotstxt v1.1.2/go.mod h1:+1AmkuG3IYkh1kv0d2qEB9Le88ehNO0zwOr3ujewlOo=
 github.com/testcontainers/testcontainers-go v0.34.0 h1:5fbgF0vIN5u+nD3IWabQwRybuB4GY8G2HHgCkbMzMHo=
 github.com/testcontainers/testcontainers-go v0.34.0/go.mod h1:6P/kMkQe8yqPHfPWNulFGdFHTD8HB2vLq/231xY2iPQ=
 github.com/testcontainers/testcontainers-go/modules/elasticsearch v0.34.0 h1:BBwJUs9xBpt1uOfO+yAr2pYW75MsyzuO/o70HTPnhe4=
@@ -833,6 +862,11 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
+golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
+golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
+golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
+golang.org/x/crypto v0.32.0/go.mod h1:ZnnJkOaASj8g0AjIduWNlq2NRxL0PlBrbKVyZ6V/Ugc=
 golang.org/x/crypto v0.46.0 h1:cKRW/pmt1pKAfetfu+RCEvjvZkA9RimPbh7bhFjGVBU=
 golang.org/x/crypto v0.46.0/go.mod h1:Evb/oLKmMraqjZ2iQTwDwvCtJkczlDuTmdJXoZVzqU0=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
@@ -864,6 +898,10 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
+golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
+golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
 golang.org/x/mod v0.30.0 h1:fDEXFVZ/fmCKProc/yAXXUijritrDzahmwwefnjoPFk=
 golang.org/x/mod v0.30.0/go.mod h1:lAsf5O2EvJeSFMiBxXDki7sCgAxEUcZHXoXMKT4GJKc=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -886,7 +924,14 @@ golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwY
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
+golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
+golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
+golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
+golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
+golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
+golang.org/x/net v0.34.0/go.mod h1:di0qlW3YNM5oh6GqDGQr92MyTozJPmybPK4Ev/Gm31k=
 golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU=
 golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@@ -904,6 +949,11 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
+golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4=
 golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -946,12 +996,24 @@ golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk=
 golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
+golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
+golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
+golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU=
+golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
+golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
+golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
+golang.org/x/term v0.28.0/go.mod h1:Sw/lC2IAUZ92udQNf3WodGtn4k/XoLyZoh8v/8uiwek=
 golang.org/x/term v0.38.0 h1:PQ5pkm/rLO6HnxFR7N2lJHOZX6Kez5Y1gDSJla6jo7Q=
 golang.org/x/term v0.38.0/go.mod h1:bSEAKrOT1W+VSu9TSCMtoGEOUcKxOKgl3LE5QEF/xVg=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -963,6 +1025,11 @@ golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
 golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
+golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
+golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
 golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU=
 golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@@ -998,6 +1065,9 @@ golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roY
 golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
+golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
+golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
+golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -1019,6 +1089,8 @@ google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7
 google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
 google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
+google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM=
+google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
 google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
@@ -1056,6 +1128,8 @@ google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQ
 google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
 google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
 google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
+google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
 google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=
 google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
diff --git a/main.go b/main.go
index d01a5fb50db9..b565789d9a8e 100644
--- a/main.go
+++ b/main.go
@@ -276,6 +276,12 @@ var (
 	jsonEnumeratorScan  = cli.Command("json-enumerator", "Find credentials from a JSON enumerator input.")
 	jsonEnumeratorPaths = jsonEnumeratorScan.Arg("path", "Path to JSON enumerator file to scan.").Strings()
 
+	webScan  = cli.Command("web", "Scan websites for leaked credentials")
+	webUrls  = webScan.Flag("urls", "One or more URLs to scan (required). Supports http:// and https://.").Required().Strings()
+	webCrawl = webScan.Flag("crawl", "Enable crawling: follow links discovered on the initial page(s).").Default("false").Bool()
+	webDepth = webScan.Flag("depth", "Maximum link depth to follow when crawling. 0 = only the seed URL(s); 1 = seed + direct links; etc.").Default("1").Int()
+	webDelay = webScan.Flag("delay", "Delay (in seconds) between requests to the same domain. Helps respect server load.").Default("1").Int()
+
 	analyzeCmd = analyzer.Command(cli)
 	usingTUI   = false
 )
@@ -1156,6 +1162,23 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics,
 		} else {
 			refs = []sources.JobProgressRef{ref}
 		}
+	case webScan.FullCommand():
+		if len(*webUrls) == 0 {
+			return scanMetrics, fmt.Errorf("invalid config: you must specify at least one url")
+		}
+
+		cfg := sources.WebConfig{
+			URLs:  *webUrls,
+			Crawl: *webCrawl,
+			Depth: *webDepth,
+			Delay: *webDelay,
+		}
+
+		if ref, err := eng.ScanWeb(ctx, cfg); err != nil {
+			return scanMetrics, fmt.Errorf("failed to scan web: %v", err)
+		} else {
+			refs = []sources.JobProgressRef{ref}
+		}
 	default:
 		return scanMetrics, fmt.Errorf("invalid command: %s", cmd)
 	}
diff --git a/pkg/engine/web.go b/pkg/engine/web.go
new file mode 100644
index 000000000000..034fccdb5d0f
--- /dev/null
+++ b/pkg/engine/web.go
@@ -0,0 +1,39 @@
+package engine
+
+import (
+	"runtime"
+
+	"google.golang.org/protobuf/proto"
+	"google.golang.org/protobuf/types/known/anypb"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/sources/web"
+)
+
+// ScanWeb scans a given web connection.
+func (e *Engine) ScanWeb(ctx context.Context, c sources.WebConfig) (sources.JobProgressRef, error) {
+	connection := &sourcespb.Web{
+		Urls:  c.URLs,
+		Crawl: c.Crawl,
+		Depth: int64(c.Depth),
+		Delay: int64(c.Delay),
+	}
+
+	var conn anypb.Any
+	err := anypb.MarshalFrom(&conn, connection, proto.MarshalOptions{})
+	if err != nil {
+		ctx.Logger().Error(err, "failed to marshal web connection")
+		return sources.JobProgressRef{}, err
+	}
+
+	sourceName := "trufflehog - web"
+	sourceID, jobID, _ := e.sourceManager.GetIDs(ctx, sourceName, web.SourceType)
+
+	webSource := &web.Source{}
+	if err := webSource.Init(ctx, sourceName, jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {
+		return sources.JobProgressRef{}, err
+	}
+	return e.sourceManager.EnumerateAndScan(ctx, sourceName, webSource)
+}
diff --git a/pkg/pb/source_metadatapb/source_metadata.pb.go b/pkg/pb/source_metadatapb/source_metadata.pb.go
index a457dee1e500..b7f39588de71 100644
--- a/pkg/pb/source_metadatapb/source_metadata.pb.go
+++ b/pkg/pb/source_metadatapb/source_metadata.pb.go
@@ -3582,6 +3582,85 @@ func (x *JSONEnumerator) GetMetadata() string {
 	return ""
 }
 
+type Web struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	Url         string `protobuf:"bytes,1,opt,name=url,proto3" json:"url,omitempty"`
+	PageTitle   string `protobuf:"bytes,2,opt,name=page_title,json=pageTitle,proto3" json:"page_title,omitempty"`
+	ContentType string `protobuf:"bytes,3,opt,name=content_type,json=contentType,proto3" json:"content_type,omitempty"`
+	Depth       int64  `protobuf:"varint,4,opt,name=depth,proto3" json:"depth,omitempty"`
+	Timestamp   string `protobuf:"bytes,5,opt,name=timestamp,proto3" json:"timestamp,omitempty"`
+}
+
+func (x *Web) Reset() {
+	*x = Web{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_source_metadata_proto_msgTypes[37]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *Web) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*Web) ProtoMessage() {}
+
+func (x *Web) ProtoReflect() protoreflect.Message {
+	mi := &file_source_metadata_proto_msgTypes[37]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use Web.ProtoReflect.Descriptor instead.
+func (*Web) Descriptor() ([]byte, []int) {
+	return file_source_metadata_proto_rawDescGZIP(), []int{37}
+}
+
+func (x *Web) GetUrl() string {
+	if x != nil {
+		return x.Url
+	}
+	return ""
+}
+
+func (x *Web) GetPageTitle() string {
+	if x != nil {
+		return x.PageTitle
+	}
+	return ""
+}
+
+func (x *Web) GetContentType() string {
+	if x != nil {
+		return x.ContentType
+	}
+	return ""
+}
+
+func (x *Web) GetDepth() int64 {
+	if x != nil {
+		return x.Depth
+	}
+	return 0
+}
+
+func (x *Web) GetTimestamp() string {
+	if x != nil {
+		return x.Timestamp
+	}
+	return ""
+}
+
 type MetaData struct {
 	state         protoimpl.MessageState
 	sizeCache     protoimpl.SizeCache
@@ -3625,13 +3704,14 @@ type MetaData struct {
 	//	*MetaData_Stdin
 	//	*MetaData_SlackContinuous
 	//	*MetaData_JsonEnumerator
+	//	*MetaData_Web
 	Data isMetaData_Data `protobuf_oneof:"data"`
 }
 
 func (x *MetaData) Reset() {
 	*x = MetaData{}
 	if protoimpl.UnsafeEnabled {
-		mi := &file_source_metadata_proto_msgTypes[37]
+		mi := &file_source_metadata_proto_msgTypes[38]
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		ms.StoreMessageInfo(mi)
 	}
@@ -3644,7 +3724,7 @@ func (x *MetaData) String() string {
 func (*MetaData) ProtoMessage() {}
 
 func (x *MetaData) ProtoReflect() protoreflect.Message {
-	mi := &file_source_metadata_proto_msgTypes[37]
+	mi := &file_source_metadata_proto_msgTypes[38]
 	if protoimpl.UnsafeEnabled && x != nil {
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		if ms.LoadMessageInfo() == nil {
@@ -3657,7 +3737,7 @@ func (x *MetaData) ProtoReflect() protoreflect.Message {
 
 // Deprecated: Use MetaData.ProtoReflect.Descriptor instead.
 func (*MetaData) Descriptor() ([]byte, []int) {
-	return file_source_metadata_proto_rawDescGZIP(), []int{37}
+	return file_source_metadata_proto_rawDescGZIP(), []int{38}
 }
 
 func (m *MetaData) GetData() isMetaData_Data {
@@ -3919,6 +3999,13 @@ func (x *MetaData) GetJsonEnumerator() *JSONEnumerator {
 	return nil
 }
 
+func (x *MetaData) GetWeb() *Web {
+	if x, ok := x.GetData().(*MetaData_Web); ok {
+		return x.Web
+	}
+	return nil
+}
+
 type isMetaData_Data interface {
 	isMetaData_Data()
 }
@@ -4067,6 +4154,10 @@ type MetaData_JsonEnumerator struct {
 	JsonEnumerator *JSONEnumerator `protobuf:"bytes,36,opt,name=jsonEnumerator,proto3,oneof"`
 }
 
+type MetaData_Web struct {
+	Web *Web `protobuf:"bytes,37,opt,name=web,proto3,oneof"`
+}
+
 func (*MetaData_Azure) isMetaData_Data() {}
 
 func (*MetaData_Bitbucket) isMetaData_Data() {}
@@ -4139,6 +4230,8 @@ func (*MetaData_SlackContinuous) isMetaData_Data() {}
 
 func (*MetaData_JsonEnumerator) isMetaData_Data() {}
 
+func (*MetaData_Web) isMetaData_Data() {}
+
 var File_source_metadata_proto protoreflect.FileDescriptor
 
 var file_source_metadata_proto_rawDesc = []byte{
@@ -4597,7 +4690,16 @@ var file_source_metadata_proto_rawDesc = []byte{
 	0x28, 0x09, 0x52, 0x0b, 0x77, 0x6f, 0x72, 0x6b, 0x73, 0x70, 0x61, 0x63, 0x65, 0x49, 0x64, 0x22,
 	0x2c, 0x0a, 0x0e, 0x4a, 0x53, 0x4f, 0x4e, 0x45, 0x6e, 0x75, 0x6d, 0x65, 0x72, 0x61, 0x74, 0x6f,
 	0x72, 0x12, 0x1a, 0x0a, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x18, 0x01, 0x20,
-	0x01, 0x28, 0x09, 0x52, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x22, 0xbf, 0x0f,
+	0x01, 0x28, 0x09, 0x52, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x22, 0x8d, 0x01,
+	0x0a, 0x03, 0x57, 0x65, 0x62, 0x12, 0x10, 0x0a, 0x03, 0x75, 0x72, 0x6c, 0x18, 0x01, 0x20, 0x01,
+	0x28, 0x09, 0x52, 0x03, 0x75, 0x72, 0x6c, 0x12, 0x1d, 0x0a, 0x0a, 0x70, 0x61, 0x67, 0x65, 0x5f,
+	0x74, 0x69, 0x74, 0x6c, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x70, 0x61, 0x67,
+	0x65, 0x54, 0x69, 0x74, 0x6c, 0x65, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e,
+	0x74, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x63, 0x6f,
+	0x6e, 0x74, 0x65, 0x6e, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x64, 0x65, 0x70,
+	0x74, 0x68, 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, 0x52, 0x05, 0x64, 0x65, 0x70, 0x74, 0x68, 0x12,
+	0x1c, 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x05, 0x20, 0x01,
+	0x28, 0x09, 0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x22, 0xe9, 0x0f,
 	0x0a, 0x08, 0x4d, 0x65, 0x74, 0x61, 0x44, 0x61, 0x74, 0x61, 0x12, 0x2e, 0x0a, 0x05, 0x61, 0x7a,
 	0x75, 0x72, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x73, 0x6f, 0x75, 0x72,
 	0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x41, 0x7a, 0x75, 0x72,
@@ -4721,45 +4823,47 @@ var file_source_metadata_proto_rawDesc = []byte{
 	0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65,
 	0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x4a, 0x53, 0x4f, 0x4e, 0x45, 0x6e, 0x75, 0x6d, 0x65,
 	0x72, 0x61, 0x74, 0x6f, 0x72, 0x48, 0x00, 0x52, 0x0e, 0x6a, 0x73, 0x6f, 0x6e, 0x45, 0x6e, 0x75,
-	0x6d, 0x65, 0x72, 0x61, 0x74, 0x6f, 0x72, 0x42, 0x06, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x2a,
-	0x3e, 0x0a, 0x0a, 0x56, 0x69, 0x73, 0x69, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x79, 0x12, 0x0a, 0x0a,
-	0x06, 0x70, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x10, 0x00, 0x12, 0x0b, 0x0a, 0x07, 0x70, 0x72, 0x69,
-	0x76, 0x61, 0x74, 0x65, 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x73, 0x68, 0x61, 0x72, 0x65, 0x64,
-	0x10, 0x02, 0x12, 0x0b, 0x0a, 0x07, 0x75, 0x6e, 0x6b, 0x6e, 0x6f, 0x77, 0x6e, 0x10, 0x03, 0x2a,
-	0xc2, 0x03, 0x0a, 0x13, 0x50, 0x6f, 0x73, 0x74, 0x6d, 0x61, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x74,
-	0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x12, 0x13, 0x0a, 0x0f, 0x55, 0x4e, 0x4b, 0x4e, 0x4f,
-	0x57, 0x4e, 0x5f, 0x50, 0x4f, 0x53, 0x54, 0x4d, 0x41, 0x4e, 0x10, 0x00, 0x12, 0x1b, 0x0a, 0x17,
-	0x52, 0x45, 0x51, 0x55, 0x45, 0x53, 0x54, 0x5f, 0x51, 0x55, 0x45, 0x52, 0x59, 0x5f, 0x50, 0x41,
-	0x52, 0x41, 0x4d, 0x45, 0x54, 0x45, 0x52, 0x10, 0x01, 0x12, 0x19, 0x0a, 0x15, 0x52, 0x45, 0x51,
-	0x55, 0x45, 0x53, 0x54, 0x5f, 0x41, 0x55, 0x54, 0x48, 0x4f, 0x52, 0x49, 0x5a, 0x41, 0x54, 0x49,
-	0x4f, 0x4e, 0x10, 0x02, 0x12, 0x12, 0x0a, 0x0e, 0x52, 0x45, 0x51, 0x55, 0x45, 0x53, 0x54, 0x5f,
-	0x48, 0x45, 0x41, 0x44, 0x45, 0x52, 0x10, 0x03, 0x12, 0x1a, 0x0a, 0x16, 0x52, 0x45, 0x51, 0x55,
-	0x45, 0x53, 0x54, 0x5f, 0x42, 0x4f, 0x44, 0x59, 0x5f, 0x46, 0x4f, 0x52, 0x4d, 0x5f, 0x44, 0x41,
-	0x54, 0x41, 0x10, 0x04, 0x12, 0x14, 0x0a, 0x10, 0x52, 0x45, 0x51, 0x55, 0x45, 0x53, 0x54, 0x5f,
-	0x42, 0x4f, 0x44, 0x59, 0x5f, 0x52, 0x41, 0x57, 0x10, 0x05, 0x12, 0x1c, 0x0a, 0x18, 0x52, 0x45,
-	0x51, 0x55, 0x45, 0x53, 0x54, 0x5f, 0x42, 0x4f, 0x44, 0x59, 0x5f, 0x55, 0x52, 0x4c, 0x5f, 0x45,
-	0x4e, 0x43, 0x4f, 0x44, 0x45, 0x44, 0x10, 0x06, 0x12, 0x18, 0x0a, 0x14, 0x52, 0x45, 0x51, 0x55,
-	0x45, 0x53, 0x54, 0x5f, 0x42, 0x4f, 0x44, 0x59, 0x5f, 0x47, 0x52, 0x41, 0x50, 0x48, 0x51, 0x4c,
-	0x10, 0x07, 0x12, 0x12, 0x0a, 0x0e, 0x52, 0x45, 0x51, 0x55, 0x45, 0x53, 0x54, 0x5f, 0x53, 0x43,
-	0x52, 0x49, 0x50, 0x54, 0x10, 0x08, 0x12, 0x0f, 0x0a, 0x0b, 0x52, 0x45, 0x51, 0x55, 0x45, 0x53,
-	0x54, 0x5f, 0x55, 0x52, 0x4c, 0x10, 0x09, 0x12, 0x18, 0x0a, 0x14, 0x45, 0x4e, 0x56, 0x49, 0x52,
-	0x4f, 0x4e, 0x4d, 0x45, 0x4e, 0x54, 0x5f, 0x56, 0x41, 0x52, 0x49, 0x41, 0x42, 0x4c, 0x45, 0x10,
-	0x0a, 0x12, 0x18, 0x0a, 0x14, 0x46, 0x4f, 0x4c, 0x44, 0x45, 0x52, 0x5f, 0x41, 0x55, 0x54, 0x48,
-	0x4f, 0x52, 0x49, 0x5a, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x10, 0x0b, 0x12, 0x11, 0x0a, 0x0d, 0x46,
-	0x4f, 0x4c, 0x44, 0x45, 0x52, 0x5f, 0x53, 0x43, 0x52, 0x49, 0x50, 0x54, 0x10, 0x0c, 0x12, 0x15,
-	0x0a, 0x11, 0x43, 0x4f, 0x4c, 0x4c, 0x45, 0x43, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x53, 0x43, 0x52,
-	0x49, 0x50, 0x54, 0x10, 0x0d, 0x12, 0x17, 0x0a, 0x13, 0x43, 0x4f, 0x4c, 0x4c, 0x45, 0x43, 0x54,
-	0x49, 0x4f, 0x4e, 0x5f, 0x56, 0x41, 0x52, 0x49, 0x41, 0x42, 0x4c, 0x45, 0x10, 0x0e, 0x12, 0x1c,
-	0x0a, 0x18, 0x43, 0x4f, 0x4c, 0x4c, 0x45, 0x43, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x41, 0x55, 0x54,
-	0x48, 0x4f, 0x52, 0x49, 0x5a, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x10, 0x0f, 0x12, 0x11, 0x0a, 0x0d,
-	0x52, 0x45, 0x53, 0x50, 0x4f, 0x4e, 0x53, 0x45, 0x5f, 0x42, 0x4f, 0x44, 0x59, 0x10, 0x10, 0x12,
-	0x13, 0x0a, 0x0f, 0x52, 0x45, 0x53, 0x50, 0x4f, 0x4e, 0x53, 0x45, 0x5f, 0x48, 0x45, 0x41, 0x44,
-	0x45, 0x52, 0x10, 0x11, 0x42, 0x43, 0x5a, 0x41, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63,
-	0x6f, 0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69,
-	0x74, 0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68, 0x6f, 0x67, 0x2f, 0x76, 0x33,
-	0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d,
-	0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f,
-	0x33,
+	0x6d, 0x65, 0x72, 0x61, 0x74, 0x6f, 0x72, 0x12, 0x28, 0x0a, 0x03, 0x77, 0x65, 0x62, 0x18, 0x25,
+	0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65,
+	0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x57, 0x65, 0x62, 0x48, 0x00, 0x52, 0x03, 0x77, 0x65,
+	0x62, 0x42, 0x06, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x2a, 0x3e, 0x0a, 0x0a, 0x56, 0x69, 0x73,
+	0x69, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x79, 0x12, 0x0a, 0x0a, 0x06, 0x70, 0x75, 0x62, 0x6c, 0x69,
+	0x63, 0x10, 0x00, 0x12, 0x0b, 0x0a, 0x07, 0x70, 0x72, 0x69, 0x76, 0x61, 0x74, 0x65, 0x10, 0x01,
+	0x12, 0x0a, 0x0a, 0x06, 0x73, 0x68, 0x61, 0x72, 0x65, 0x64, 0x10, 0x02, 0x12, 0x0b, 0x0a, 0x07,
+	0x75, 0x6e, 0x6b, 0x6e, 0x6f, 0x77, 0x6e, 0x10, 0x03, 0x2a, 0xc2, 0x03, 0x0a, 0x13, 0x50, 0x6f,
+	0x73, 0x74, 0x6d, 0x61, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70,
+	0x65, 0x12, 0x13, 0x0a, 0x0f, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x5f, 0x50, 0x4f, 0x53,
+	0x54, 0x4d, 0x41, 0x4e, 0x10, 0x00, 0x12, 0x1b, 0x0a, 0x17, 0x52, 0x45, 0x51, 0x55, 0x45, 0x53,
+	0x54, 0x5f, 0x51, 0x55, 0x45, 0x52, 0x59, 0x5f, 0x50, 0x41, 0x52, 0x41, 0x4d, 0x45, 0x54, 0x45,
+	0x52, 0x10, 0x01, 0x12, 0x19, 0x0a, 0x15, 0x52, 0x45, 0x51, 0x55, 0x45, 0x53, 0x54, 0x5f, 0x41,
+	0x55, 0x54, 0x48, 0x4f, 0x52, 0x49, 0x5a, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x10, 0x02, 0x12, 0x12,
+	0x0a, 0x0e, 0x52, 0x45, 0x51, 0x55, 0x45, 0x53, 0x54, 0x5f, 0x48, 0x45, 0x41, 0x44, 0x45, 0x52,
+	0x10, 0x03, 0x12, 0x1a, 0x0a, 0x16, 0x52, 0x45, 0x51, 0x55, 0x45, 0x53, 0x54, 0x5f, 0x42, 0x4f,
+	0x44, 0x59, 0x5f, 0x46, 0x4f, 0x52, 0x4d, 0x5f, 0x44, 0x41, 0x54, 0x41, 0x10, 0x04, 0x12, 0x14,
+	0x0a, 0x10, 0x52, 0x45, 0x51, 0x55, 0x45, 0x53, 0x54, 0x5f, 0x42, 0x4f, 0x44, 0x59, 0x5f, 0x52,
+	0x41, 0x57, 0x10, 0x05, 0x12, 0x1c, 0x0a, 0x18, 0x52, 0x45, 0x51, 0x55, 0x45, 0x53, 0x54, 0x5f,
+	0x42, 0x4f, 0x44, 0x59, 0x5f, 0x55, 0x52, 0x4c, 0x5f, 0x45, 0x4e, 0x43, 0x4f, 0x44, 0x45, 0x44,
+	0x10, 0x06, 0x12, 0x18, 0x0a, 0x14, 0x52, 0x45, 0x51, 0x55, 0x45, 0x53, 0x54, 0x5f, 0x42, 0x4f,
+	0x44, 0x59, 0x5f, 0x47, 0x52, 0x41, 0x50, 0x48, 0x51, 0x4c, 0x10, 0x07, 0x12, 0x12, 0x0a, 0x0e,
+	0x52, 0x45, 0x51, 0x55, 0x45, 0x53, 0x54, 0x5f, 0x53, 0x43, 0x52, 0x49, 0x50, 0x54, 0x10, 0x08,
+	0x12, 0x0f, 0x0a, 0x0b, 0x52, 0x45, 0x51, 0x55, 0x45, 0x53, 0x54, 0x5f, 0x55, 0x52, 0x4c, 0x10,
+	0x09, 0x12, 0x18, 0x0a, 0x14, 0x45, 0x4e, 0x56, 0x49, 0x52, 0x4f, 0x4e, 0x4d, 0x45, 0x4e, 0x54,
+	0x5f, 0x56, 0x41, 0x52, 0x49, 0x41, 0x42, 0x4c, 0x45, 0x10, 0x0a, 0x12, 0x18, 0x0a, 0x14, 0x46,
+	0x4f, 0x4c, 0x44, 0x45, 0x52, 0x5f, 0x41, 0x55, 0x54, 0x48, 0x4f, 0x52, 0x49, 0x5a, 0x41, 0x54,
+	0x49, 0x4f, 0x4e, 0x10, 0x0b, 0x12, 0x11, 0x0a, 0x0d, 0x46, 0x4f, 0x4c, 0x44, 0x45, 0x52, 0x5f,
+	0x53, 0x43, 0x52, 0x49, 0x50, 0x54, 0x10, 0x0c, 0x12, 0x15, 0x0a, 0x11, 0x43, 0x4f, 0x4c, 0x4c,
+	0x45, 0x43, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x53, 0x43, 0x52, 0x49, 0x50, 0x54, 0x10, 0x0d, 0x12,
+	0x17, 0x0a, 0x13, 0x43, 0x4f, 0x4c, 0x4c, 0x45, 0x43, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x56, 0x41,
+	0x52, 0x49, 0x41, 0x42, 0x4c, 0x45, 0x10, 0x0e, 0x12, 0x1c, 0x0a, 0x18, 0x43, 0x4f, 0x4c, 0x4c,
+	0x45, 0x43, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x41, 0x55, 0x54, 0x48, 0x4f, 0x52, 0x49, 0x5a, 0x41,
+	0x54, 0x49, 0x4f, 0x4e, 0x10, 0x0f, 0x12, 0x11, 0x0a, 0x0d, 0x52, 0x45, 0x53, 0x50, 0x4f, 0x4e,
+	0x53, 0x45, 0x5f, 0x42, 0x4f, 0x44, 0x59, 0x10, 0x10, 0x12, 0x13, 0x0a, 0x0f, 0x52, 0x45, 0x53,
+	0x50, 0x4f, 0x4e, 0x53, 0x45, 0x5f, 0x48, 0x45, 0x41, 0x44, 0x45, 0x52, 0x10, 0x11, 0x42, 0x43,
+	0x5a, 0x41, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x72, 0x75,
+	0x66, 0x66, 0x6c, 0x65, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x2f, 0x74, 0x72, 0x75,
+	0x66, 0x66, 0x6c, 0x65, 0x68, 0x6f, 0x67, 0x2f, 0x76, 0x33, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70,
+	0x62, 0x2f, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74,
+	0x61, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
 }
 
 var (
@@ -4775,7 +4879,7 @@ func file_source_metadata_proto_rawDescGZIP() []byte {
 }
 
 var file_source_metadata_proto_enumTypes = make([]protoimpl.EnumInfo, 2)
-var file_source_metadata_proto_msgTypes = make([]protoimpl.MessageInfo, 38)
+var file_source_metadata_proto_msgTypes = make([]protoimpl.MessageInfo, 39)
 var file_source_metadata_proto_goTypes = []interface{}{
 	(Visibility)(0),               // 0: source_metadata.Visibility
 	(PostmanLocationType)(0),      // 1: source_metadata.PostmanLocationType
@@ -4816,8 +4920,9 @@ var file_source_metadata_proto_goTypes = []interface{}{
 	(*Stdin)(nil),                 // 36: source_metadata.Stdin
 	(*SlackContinuous)(nil),       // 37: source_metadata.SlackContinuous
 	(*JSONEnumerator)(nil),        // 38: source_metadata.JSONEnumerator
-	(*MetaData)(nil),              // 39: source_metadata.MetaData
-	(*timestamppb.Timestamp)(nil), // 40: google.protobuf.Timestamp
+	(*Web)(nil),                   // 39: source_metadata.Web
+	(*MetaData)(nil),              // 40: source_metadata.MetaData
+	(*timestamppb.Timestamp)(nil), // 41: google.protobuf.Timestamp
 }
 var file_source_metadata_proto_depIdxs = []int32{
 	0,  // 0: source_metadata.Github.visibility:type_name -> source_metadata.Visibility
@@ -4828,7 +4933,7 @@ var file_source_metadata_proto_depIdxs = []int32{
 	18, // 5: source_metadata.Forager.pypi:type_name -> source_metadata.PyPi
 	0,  // 6: source_metadata.AzureRepos.visibility:type_name -> source_metadata.Visibility
 	1,  // 7: source_metadata.Postman.location_type:type_name -> source_metadata.PostmanLocationType
-	40, // 8: source_metadata.Vector.timestamp:type_name -> google.protobuf.Timestamp
+	41, // 8: source_metadata.Vector.timestamp:type_name -> google.protobuf.Timestamp
 	32, // 9: source_metadata.Webhook.vector:type_name -> source_metadata.Vector
 	0,  // 10: source_metadata.SlackContinuous.visibility:type_name -> source_metadata.Visibility
 	2,  // 11: source_metadata.MetaData.azure:type_name -> source_metadata.Azure
@@ -4867,11 +4972,12 @@ var file_source_metadata_proto_depIdxs = []int32{
 	36, // 44: source_metadata.MetaData.stdin:type_name -> source_metadata.Stdin
 	37, // 45: source_metadata.MetaData.slackContinuous:type_name -> source_metadata.SlackContinuous
 	38, // 46: source_metadata.MetaData.jsonEnumerator:type_name -> source_metadata.JSONEnumerator
-	47, // [47:47] is the sub-list for method output_type
-	47, // [47:47] is the sub-list for method input_type
-	47, // [47:47] is the sub-list for extension type_name
-	47, // [47:47] is the sub-list for extension extendee
-	0,  // [0:47] is the sub-list for field type_name
+	39, // 47: source_metadata.MetaData.web:type_name -> source_metadata.Web
+	48, // [48:48] is the sub-list for method output_type
+	48, // [48:48] is the sub-list for method input_type
+	48, // [48:48] is the sub-list for extension type_name
+	48, // [48:48] is the sub-list for extension extendee
+	0,  // [0:48] is the sub-list for field type_name
 }
 
 func init() { file_source_metadata_proto_init() }
@@ -5325,6 +5431,18 @@ func file_source_metadata_proto_init() {
 			}
 		}
 		file_source_metadata_proto_msgTypes[37].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*Web); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_source_metadata_proto_msgTypes[38].Exporter = func(v interface{}, i int) interface{} {
 			switch v := v.(*MetaData); i {
 			case 0:
 				return &v.state
@@ -5345,7 +5463,7 @@ func file_source_metadata_proto_init() {
 	file_source_metadata_proto_msgTypes[31].OneofWrappers = []interface{}{
 		(*Webhook_Vector)(nil),
 	}
-	file_source_metadata_proto_msgTypes[37].OneofWrappers = []interface{}{
+	file_source_metadata_proto_msgTypes[38].OneofWrappers = []interface{}{
 		(*MetaData_Azure)(nil),
 		(*MetaData_Bitbucket)(nil),
 		(*MetaData_Circleci)(nil),
@@ -5382,6 +5500,7 @@ func file_source_metadata_proto_init() {
 		(*MetaData_Stdin)(nil),
 		(*MetaData_SlackContinuous)(nil),
 		(*MetaData_JsonEnumerator)(nil),
+		(*MetaData_Web)(nil),
 	}
 	type x struct{}
 	out := protoimpl.TypeBuilder{
@@ -5389,7 +5508,7 @@ func file_source_metadata_proto_init() {
 			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
 			RawDescriptor: file_source_metadata_proto_rawDesc,
 			NumEnums:      2,
-			NumMessages:   38,
+			NumMessages:   39,
 			NumExtensions: 0,
 			NumServices:   0,
 		},
diff --git a/pkg/pb/source_metadatapb/source_metadata.pb.validate.go b/pkg/pb/source_metadatapb/source_metadata.pb.validate.go
index 1cb13f369b59..26f77c28dc14 100644
--- a/pkg/pb/source_metadatapb/source_metadata.pb.validate.go
+++ b/pkg/pb/source_metadatapb/source_metadata.pb.validate.go
@@ -4349,6 +4349,114 @@ var _ interface {
 	ErrorName() string
 } = JSONEnumeratorValidationError{}
 
+// Validate checks the field values on Web with the rules defined in the proto
+// definition for this message. If any rules are violated, the first error
+// encountered is returned, or nil if there are no violations.
+func (m *Web) Validate() error {
+	return m.validate(false)
+}
+
+// ValidateAll checks the field values on Web with the rules defined in the
+// proto definition for this message. If any rules are violated, the result is
+// a list of violation errors wrapped in WebMultiError, or nil if none found.
+func (m *Web) ValidateAll() error {
+	return m.validate(true)
+}
+
+func (m *Web) validate(all bool) error {
+	if m == nil {
+		return nil
+	}
+
+	var errors []error
+
+	// no validation rules for Url
+
+	// no validation rules for PageTitle
+
+	// no validation rules for ContentType
+
+	// no validation rules for Depth
+
+	// no validation rules for Timestamp
+
+	if len(errors) > 0 {
+		return WebMultiError(errors)
+	}
+
+	return nil
+}
+
+// WebMultiError is an error wrapping multiple validation errors returned by
+// Web.ValidateAll() if the designated constraints aren't met.
+type WebMultiError []error
+
+// Error returns a concatenation of all the error messages it wraps.
+func (m WebMultiError) Error() string {
+	var msgs []string
+	for _, err := range m {
+		msgs = append(msgs, err.Error())
+	}
+	return strings.Join(msgs, "; ")
+}
+
+// AllErrors returns a list of validation violation errors.
+func (m WebMultiError) AllErrors() []error { return m }
+
+// WebValidationError is the validation error returned by Web.Validate if the
+// designated constraints aren't met.
+type WebValidationError struct {
+	field  string
+	reason string
+	cause  error
+	key    bool
+}
+
+// Field function returns field value.
+func (e WebValidationError) Field() string { return e.field }
+
+// Reason function returns reason value.
+func (e WebValidationError) Reason() string { return e.reason }
+
+// Cause function returns cause value.
+func (e WebValidationError) Cause() error { return e.cause }
+
+// Key function returns key value.
+func (e WebValidationError) Key() bool { return e.key }
+
+// ErrorName returns error name.
+func (e WebValidationError) ErrorName() string { return "WebValidationError" }
+
+// Error satisfies the builtin error interface
+func (e WebValidationError) Error() string {
+	cause := ""
+	if e.cause != nil {
+		cause = fmt.Sprintf(" | caused by: %v", e.cause)
+	}
+
+	key := ""
+	if e.key {
+		key = "key for "
+	}
+
+	return fmt.Sprintf(
+		"invalid %sWeb.%s: %s%s",
+		key,
+		e.field,
+		e.reason,
+		cause)
+}
+
+var _ error = WebValidationError{}
+
+var _ interface {
+	Field() string
+	Reason() string
+	Key() bool
+	Cause() error
+	ErrorName() string
+} = WebValidationError{}
+
 // Validate checks the field values on MetaData with the rules defined in the
 // proto definition for this message. If any rules are violated, the first
 // error encountered is returned, or nil if there are no violations.
@@ -5848,6 +5956,47 @@ func (m *MetaData) validate(all bool) error {
 			}
 		}
 
+	case *MetaData_Web:
+		if v == nil {
+			err := MetaDataValidationError{
+				field:  "Data",
+				reason: "oneof value cannot be a typed-nil",
+			}
+			if !all {
+				return err
+			}
+			errors = append(errors, err)
+		}
+
+		if all {
+			switch v := interface{}(m.GetWeb()).(type) {
+			case interface{ ValidateAll() error }:
+				if err := v.ValidateAll(); err != nil {
+					errors = append(errors, MetaDataValidationError{
+						field:  "Web",
+						reason: "embedded message failed validation",
+						cause:  err,
+					})
+				}
+			case interface{ Validate() error }:
+				if err := v.Validate(); err != nil {
+					errors = append(errors, MetaDataValidationError{
+						field:  "Web",
+						reason: "embedded message failed validation",
+						cause:  err,
+					})
+				}
+			}
+		} else if v, ok := interface{}(m.GetWeb()).(interface{ Validate() error }); ok {
+			if err := v.Validate(); err != nil {
+				return MetaDataValidationError{
+					field:  "Web",
+					reason: "embedded message failed validation",
+					cause:  err,
+				}
+			}
+		}
+
 	default:
 		_ = v // ensures v is used
 	}
diff --git a/pkg/sources/web/web.go b/pkg/sources/web/web.go
index adc75af386d4..fb7fc5d916ac 100644
--- a/pkg/sources/web/web.go
+++ b/pkg/sources/web/web.go
@@ -1,10 +1,24 @@
 package web
 
 import (
+	"bytes"
+	"errors"
+	"fmt"
+	"net/url"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/gocolly/colly/v2"
+	"golang.org/x/net/html"
+	"google.golang.org/protobuf/proto"
+	"google.golang.org/protobuf/types/known/anypb"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
-	"google.golang.org/protobuf/types/known/anypb"
 )
 
 const SourceType = sourcespb.SourceType_SOURCE_TYPE_WEB
@@ -33,10 +47,165 @@ func (s *Source) JobID() sources.JobID       { return s.jobId }
 func (s *Source) Init(ctx context.Context, name string, jobId sources.JobID, sourceId sources.SourceID,
 	verify bool, connection *anypb.Any, concurrency int,
 ) error {
+	s.name = name
+	s.sourceId = sourceId
+	s.jobId = jobId
+	s.verify = verify
+	s.concurrency = concurrency
+	// If s.concurrency is 0, use 1
+	if s.concurrency == 0 {
+		s.concurrency = 1
+	}
+
+	if err := anypb.UnmarshalTo(connection, &s.conn, proto.UnmarshalOptions{}); err != nil {
+		return fmt.Errorf("error unmarshalling connection: %w", err)
+	}
+
+	// validations
+	if len(s.conn.GetUrls()) == 0 {
+		return errors.New("no URL provided")
+	}
+	// TODO: Enable support for more than one URLs
+	if len(s.conn.GetUrls()) > 1 {
+		return errors.New("only one base URL is allowed right now")
+	}
+
+	// Validate URLs format
+	for _, u := range s.conn.GetUrls() {
+		if _, err := url.Parse(u); err != nil {
+			return fmt.Errorf("invalid URL %q: %w", u, err)
+		}
+	}
+
+	// TODO: reset metrics if needed
+
 	return nil
 }
 
 // Chunks emits data over a channel that is decoded and scanned for secrets.
 func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ ...sources.ChunkingTarget) error {
+	var wg sync.WaitGroup
+
+	// Create a background context for crawling (independent of incoming ctx)
+	crawlCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	for _, url := range s.conn.GetUrls() {
+		ctx.Logger().V(5).Info("Processing Url", "url", url)
+		wg.Add(1)
+		go func(url string) {
+			defer wg.Done()
+			s.crawlURL(crawlCtx, url, chunksChan)
+		}(url)
+	}
+
+	// Block until all crawls complete
+	wg.Wait()
+	ctx.Logger().Info("All crawls completed")
 	return nil
 }
+
+func (s *Source) crawlURL(ctx context.Context, seedURL string, chunksChan chan *sources.Chunk) error {
+	url, err := url.Parse(seedURL)
+	if err != nil {
+		return fmt.Errorf("invalid URL %q: %w", seedURL, err)
+	}
+
+	collector := colly.NewCollector(
+		colly.UserAgent("trufflehog-web (+https://github.com/trufflesecurity/trufflehog)"),
+		colly.AllowedDomains(url.Hostname(), fmt.Sprintf("*.%s", url.Hostname())), // with subdomains
+		colly.Async(true),
+	)
+
+	// Respect robots.txt
+	// TODO: maybe allow users to set this as well with warning
+	collector.IgnoreRobotsTxt = false
+
+	collector.Limit(&colly.LimitRule{
+		DomainGlob:  "*",
+		Parallelism: s.concurrency,
+		Delay:       time.Duration(s.conn.GetDelay()) * time.Second,
+	})
+
+	// Set up callbacks
+	collector.OnResponse(func(r *colly.Response) {
+		ctx.Logger().Info("OnResponse fired", "url", r.Request.URL)
+		if err := s.processChunk(ctx, r, chunksChan); err != nil {
+			ctx.Logger().Error(err, "error processing page")
+		}
+	})
+	collector.OnError(func(r *colly.Response, err error) {
+		ctx.Logger().Error(err, "error fetching page", "url", r.Request.URL)
+	})
+
+	// Create a channel to signal when the crawl is done.
+	done := make(chan struct{})
+	go func() {
+		ctx.Logger().Info("starting crawl")
+		if err := collector.Visit(seedURL); err != nil {
+			ctx.Logger().Error(err, "Visit failed")
+		}
+		collector.Wait() // blocks until all requests finish
+		close(done)
+	}()
+
+	// Wait for either crawl to finish or context cancellation.
+	select {
+	case <-done:
+		ctx.Logger().Info("crawl finished normally")
+		return nil
+	case <-ctx.Done():
+		ctx.Logger().Info("context cancelled or timeout reached")
+		<-done // Wait for goroutine to finish cleanup
+		return ctx.Err()
+	}
+}
+
+func (s *Source) processChunk(ctx context.Context, data *colly.Response, chunksChan chan *sources.Chunk) error {
+	pageTitle := extractPageTitle(data.Body)
+
+	ctx.Logger().V(5).WithValues("page_title", pageTitle).Info("Processing web chunk")
+
+	// Create a chunk from the response body.
+	chunk := &sources.Chunk{
+		Data:         data.Body,
+		SourceType:   s.Type(),
+		SourceName:   s.name,
+		SourceID:     s.SourceID(),
+		JobID:        s.JobID(),
+		SourceVerify: s.verify,
+		SourceMetadata: &source_metadatapb.MetaData{
+			Data: &source_metadatapb.MetaData_Web{
+				Web: &source_metadatapb.Web{
+					Url:         data.Request.URL.String(),
+					PageTitle:   pageTitle,
+					Depth:       int64(data.Request.Depth),
+					ContentType: data.Headers.Get("Content-Type"),
+					Timestamp:   time.Now().UTC().Format(time.RFC3339),
+				},
+			},
+		},
+	}
+
+	return common.CancellableWrite(ctx, chunksChan, chunk)
+}
+
+func extractPageTitle(body []byte) string {
+	doc, err := html.Parse(bytes.NewReader(body))
+	if err != nil {
+		return ""
+	}
+	var title string
+	var f func(*html.Node)
+	f = func(n *html.Node) {
+		if n.Type == html.ElementNode && n.Data == "title" && n.FirstChild != nil {
+			title = strings.TrimSpace(n.FirstChild.Data)
+			return
+		}
+		for c := n.FirstChild; c != nil; c = c.NextSibling {
+			f(c)
+		}
+	}
+	f(doc)
+	return title
+}
diff --git a/proto/source_metadata.proto b/proto/source_metadata.proto
index 40846c1f7e04..2916f9e7348a 100644
--- a/proto/source_metadata.proto
+++ b/proto/source_metadata.proto
@@ -392,6 +392,14 @@ message JSONEnumerator {
   string metadata = 1;
 }
 
+message Web {
+  string url = 1;
+  string page_title = 2;
+  string content_type = 3;
+  int64 depth = 4;
+  string timestamp = 5;
+}
+
 message MetaData {
   oneof data {
     Azure azure = 1;
@@ -430,5 +438,6 @@ message MetaData {
     Stdin stdin = 34;
     SlackContinuous slackContinuous = 35;
     JSONEnumerator jsonEnumerator = 36;
+    Web web = 37;
   }
 }

From c46eca1e4109a95632741ff53b9290e718839d58 Mon Sep 17 00:00:00 2001
From: Kashif Khan <kashif.khan@trufflesec.com>
Date: Fri, 27 Mar 2026 20:10:43 +0500
Subject: [PATCH 03/20] some more enhancements + README.md

---
 main.go                   |   2 +-
 pkg/sources/web/README.md | 242 ++++++++++++++++++++++++++++++++++++++
 2 files changed, 243 insertions(+), 1 deletion(-)
 create mode 100644 pkg/sources/web/README.md

diff --git a/main.go b/main.go
index b565789d9a8e..c8d3fe33842c 100644
--- a/main.go
+++ b/main.go
@@ -277,7 +277,7 @@ var (
 	jsonEnumeratorPaths = jsonEnumeratorScan.Arg("path", "Path to JSON enumerator file to scan.").Strings()
 
 	webScan  = cli.Command("web", "Scan websites for leaked credentials")
-	webUrls  = webScan.Flag("urls", "One or more URLs to scan (required). Supports http:// and https://.").Required().Strings()
+	webUrls  = webScan.Flag("url", "One or more URLs to scan (required). You can repeat this flag. Supports http:// and https://.").Required().Strings()
 	webCrawl = webScan.Flag("crawl", "Enable crawling: follow links discovered on the initial page(s).").Default("false").Bool()
 	webDepth = webScan.Flag("depth", "Maximum link depth to follow when crawling. 0 = only the seed URL(s); 1 = seed + direct links; etc.").Default("1").Int()
 	webDelay = webScan.Flag("delay", "Delay (in seconds) between requests to the same domain. Helps respect server load.").Default("1").Int()
diff --git a/pkg/sources/web/README.md b/pkg/sources/web/README.md
new file mode 100644
index 000000000000..0f32075c98e8
--- /dev/null
+++ b/pkg/sources/web/README.md
@@ -0,0 +1,242 @@
+# TruffleHog Web Source
+
+The Web source enables TruffleHog to crawl and scan websites for secrets and sensitive information. It uses the Colly web scraper framework to systematically browse web pages and analyze their content for exposed credentials, API keys, private keys, and other secrets.
+
+## Features
+
+- **Web Crawling**: Automatically crawl websites starting from a seed URL
+- **Robots.txt Compliance**: Respects website `robots.txt` rules for ethical crawling
+- **Subdomain Support**: Crawls subdomains of the target domain
+- **Customizable Delays**: Set delays between requests to avoid overwhelming servers
+- **Metadata Extraction**: Captures page titles, URLs, content types, and timestamps
+- **Error Handling**: Gracefully handles network errors and HTTP failures
+
+## Configuration
+
+### Required Parameters
+
+- **`--url`**: One or more URLs to scan (required)
+  - Supports both `http://` and `https://` URLs
+  - Examples: `https://example.com` or `http://staging.app.com`
+  - Can specify multiple URLs: `--url https://example.com --url https://app.com`
+
+### Optional Parameters
+
+- **`--crawl`**: Enable crawling to follow links discovered on pages (default: `false`)
+  - `false`: Only scan the provided seed URL(s), don't follow links
+  - `true`: Follow discovered links to scan additional pages
+  - Useful for comprehensive scanning of entire websites
+
+- **`--depth`**: Maximum link depth to follow when crawling (default: `1`)
+  - `0`: Only scan the seed URL(s), no link following
+  - `1`: Scan seed URL(s) + direct links from those pages
+  - `2`: Scan seed + direct links + links from those pages (two levels deep)
+  - `3+`: Continue following links up to the specified depth
+  - Note: Deeper scans take longer and consume more resources
+
+- **`--delay`**: Delay in seconds between requests to the same domain (default: `1`)
+  - Recommended: 1-2 seconds for responsible, server-friendly scanning
+  - Helps avoid overwhelming the target website
+  - Respects `robots.txt` Crawl-delay directives when present
+
+## Usage
+
+### Command Line Examples
+
+**Scan single URL only (no crawling)**
+```bash
+trufflehog web --url https://example.com
+```
+
+**Scan single URL with 1 level of link following**
+```bash
+trufflehog web --url https://example.com --crawl --depth 1 --delay 2
+```
+
+**Scan multiple URLs with deeper crawling**
+```bash
+trufflehog web \
+  --url https://example.com \
+  --url https://app.example.com \
+  --crawl \
+  --depth 2 \
+  --delay 1
+```
+
+**Comprehensive website scan (2 levels deep, 2-second delays)**
+```bash
+trufflehog web --url https://mycompany.com --crawl --depth 2 --delay 2
+```
+
+## Behavior
+
+### Domain Handling
+
+- Crawls the exact domain provided (e.g., `example.com`)
+- Crawls all subdomains (e.g., `www.example.com`, `mail.example.com`)
+- Does NOT crawl other domains or external links
+
+### Robots.txt Respect
+
+By default, the crawler respects `robots.txt` files:
+- Reads `robots.txt` from the website root
+- Skips paths marked as disallowed
+- Honors crawl-delay directives
+
+To ignore `robots.txt` (not recommended), modify the code:
+```go
+collector.IgnoreRobotsTxt = true
+```
+
+### User Agent
+
+The crawler identifies itself as:
+```
+trufflehog-web (+https://github.com/trufflesecurity/trufflehog)
+```
+
+This allows website administrators to identify TruffleHog requests in their logs.
+
+## Example Scenarios
+
+### Quick Scan - Check Single Page for Secrets
+
+```bash
+trufflehog web --url https://mycompany.com
+```
+- Scans only the homepage
+- No link following
+- 1 second delay between any requests
+
+### Thorough Scan - Crawl Entire Website
+
+```bash
+trufflehog web \
+  --url https://mycompany.com \
+  --crawl \
+  --depth 2 \
+  --delay 2
+```
+- Starts from homepage
+- Follows links up to 2 levels deep
+- Respectful 2-second delays between requests
+
+### Multi-Site Scan - Check Multiple URLs
+
+```bash
+trufflehog web \
+  --url https://main.company.com \
+  --url https://staging.company.com \
+  --url https://api.company.com \
+  --crawl \
+  --depth 1 \
+  --delay 1
+```
+- Scans 3 different URLs
+- Follows direct links from each
+- 1 second delay to keep scanning fast
+
+## Best Practices
+
+1. **Always Get Permission**: Only scan websites you own or have explicit permission to scan
+
+2. **Start Conservative**: Begin with no crawling, then gradually increase depth if needed
+
+3. **Use Appropriate Delays**: 
+   - `--delay 1`: Good for most websites
+   - `--delay 2`: Large or busy websites
+   - `--delay 0.5`: Staging/internal websites only
+
+4. **Respect Crawl Depth**: 
+   - `--depth 0`: Just the seed URL (fastest, least coverage)
+   - `--depth 1`: Seed + direct links (balanced)
+   - `--depth 2+`: Comprehensive but slower and more resource-intensive
+
+5. **Monitor Robot Rules**: Keep `robots.txt` respect enabled to honor website crawling guidelines
+
+6. **Check Logs**: Review output to ensure scanning is working as expected
+
+7. **Test First**: Test on staging environments before scanning production sites
+
+## Output
+
+The Web source emits chunks containing:
+
+- **Page Content**: The raw HTML/text content of each page
+- **Page Title**: Extracted from the `<title>` tag
+- **URL**: The full URL of the crawled page
+- **Depth**: How many links deep the page is from the seed URL
+- **Content-Type**: The MIME type of the content (e.g., `text/html`)
+- **Timestamp**: When the page was crawled (UTC, RFC3339 format)
+
+### Example Metadata
+
+```json
+{
+  "url": "https://example.com/about",
+  "page_title": "About Us",
+  "depth": 1,
+  "content_type": "text/html; charset=utf-8",
+  "timestamp": "2026-03-27T17:26:34Z"
+}
+```
+
+## Limitations
+
+- **Link Depth Only**: Maximum crawl depth is limited by the `--depth` flag
+  - Deeper scans take longer and consume more memory
+  - Very deep crawls (5+) on large websites may timeout or consume excessive resources
+
+- **Single Domain**: Only crawls the target domain and its subdomains
+  - External links are skipped by design
+  - Run multiple scans for different domains
+
+- **30-Second Timeout**: Hard limit on individual crawl operations
+  - Adjust in code if needed: `context.WithTimeout(context.Background(), 30*time.Second)`
+
+- **No JavaScript Rendering**: Static HTML content only
+  - Websites with JavaScript-rendered content may appear incomplete
+  - Future enhancement: Add JavaScript rendering support
+
+- **No Authentication**: Cannot scan behind login pages
+  - Workaround: Manually extract session cookies and pass them as headers
+  - Future enhancement: Add authentication support
+
+## Troubleshooting
+
+### No Pages Crawled
+
+Check for:
+1. **Invalid URL**: Ensure the URL is valid and accessible
+2. **Network Issues**: Verify internet connectivity
+3. **Robots.txt Block**: The website's `robots.txt` may block all crawling
+4. **No Discoverable Links**: The page may have no links for the crawler to follow
+
+### Slow Crawling
+
+- Increase `concurrency` (but be respectful)
+- Reduce `delay` if appropriate
+- Check your internet connection
+
+## Security Considerations
+
+- **Sensitive Data**: Be cautious when scanning internal or staging environments
+- **Legal Compliance**: Ensure you have authorization before scanning websites
+- **Network Traffic**: Crawling generates significant network traffic and server logs
+
+## Future Enhancements
+
+- [ ] JavaScript rendering support (Puppeteer/Playwright integration)
+- [ ] Authentication support (Basic auth, cookies, form login)
+- [ ] Custom header configuration
+- [ ] Form submission and POST request handling
+- [ ] Incremental crawling with state persistence
+- [ ] Configurable timeout per scan
+- [ ] Rate limiting by content size
+- [ ] Proxy support for scanning through corporate networks
+
+## References
+
+- [Colly Web Scraping Framework](http://go-colly.org/)
+- [Robots.txt Specification](https://en.wikipedia.org/wiki/Robots.txt)
+- [Responsible Web Crawling Guidelines](https://www.robotstxt.org/)

From 015086079a25c10c64fd178fbff98938c51c2803 Mon Sep 17 00:00:00 2001
From: Kashif Khan <kashif.khan@trufflesec.com>
Date: Fri, 27 Mar 2026 21:04:33 +0500
Subject: [PATCH 04/20] A simple working test

---
 pkg/sources/web/web_test.go | 76 +++++++++++++++++++++++++++++++++++++
 1 file changed, 76 insertions(+)
 create mode 100644 pkg/sources/web/web_test.go

diff --git a/pkg/sources/web/web_test.go b/pkg/sources/web/web_test.go
new file mode 100644
index 000000000000..8ed839f8bdcb
--- /dev/null
+++ b/pkg/sources/web/web_test.go
@@ -0,0 +1,76 @@
+package web
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"sync"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"google.golang.org/protobuf/types/known/anypb"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
+)
+
+func TestWebSource_HappyPath(t *testing.T) {
+	// Create a test server that returns a simple HTML page.
+	testServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "text/html; charset=utf-8")
+		w.Write([]byte(`<!DOCTYPE html><html><head><title>Test Page</title></head><body>Hello, world!</body></html>`))
+	}))
+	defer testServer.Close()
+
+	// Build the web source configuration.
+	webConfig := &sourcespb.Web{
+		Urls:  []string{testServer.URL},
+		Crawl: false,
+		Depth: 0,
+		Delay: 0,
+	}
+
+	conn := &anypb.Any{}
+	err := conn.MarshalFrom(webConfig)
+	assert.NoError(t, err)
+
+	s := &Source{}
+	err = s.Init(context.TODO(), "test source", 0, 0, false, conn, 1)
+	assert.NoError(t, err)
+
+	var wg sync.WaitGroup
+	chunksChan := make(chan *sources.Chunk, 1)
+	chunkCounter := 0
+
+	// Collect all chunks.
+	var chunks []*sources.Chunk
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		for chunk := range chunksChan {
+			assert.NotEmpty(t, chunk)
+			chunkCounter++
+			chunks = append(chunks, chunk)
+		}
+	}()
+
+	err = s.Chunks(context.TODO(), chunksChan)
+	assert.NoError(t, err)
+
+	close(chunksChan)
+	wg.Wait()
+
+	assert.Equal(t, 1, chunkCounter)
+	chunk := chunks[0]
+
+	// Check the chunk data.
+	assert.Contains(t, string(chunk.Data), "Hello, world!")
+	// Verify the metadata.
+	meta, ok := chunk.SourceMetadata.Data.(*source_metadatapb.MetaData_Web)
+	assert.True(t, ok, "expected web metadata")
+	assert.Equal(t, "Test Page", meta.Web.PageTitle)
+	assert.Equal(t, "text/html; charset=utf-8", meta.Web.ContentType)
+	assert.Equal(t, int64(1), meta.Web.Depth) // default 1 depth
+	assert.NotEmpty(t, meta.Web.Timestamp)
+}

From 9448016e54fcc04bf5498b44d6f12744b75f5396 Mon Sep 17 00:00:00 2001
From: Kashif Khan <kashif.khan@trufflesec.com>
Date: Mon, 30 Mar 2026 12:57:09 +0500
Subject: [PATCH 05/20] user-agent flag

---
 main.go                                 |  25 ++-
 pkg/pb/sourcespb/sources.pb.go          | 210 +++++++++++++-----------
 pkg/pb/sourcespb/sources.pb.validate.go |   2 +
 pkg/sources/sources.go                  |   9 +-
 pkg/sources/web/web.go                  |   8 +-
 proto/sources.proto                     |   1 +
 6 files changed, 141 insertions(+), 114 deletions(-)

diff --git a/main.go b/main.go
index c8d3fe33842c..47189dc73630 100644
--- a/main.go
+++ b/main.go
@@ -276,11 +276,12 @@ var (
 	jsonEnumeratorScan  = cli.Command("json-enumerator", "Find credentials from a JSON enumerator input.")
 	jsonEnumeratorPaths = jsonEnumeratorScan.Arg("path", "Path to JSON enumerator file to scan.").Strings()
 
-	webScan  = cli.Command("web", "Scan websites for leaked credentials")
-	webUrls  = webScan.Flag("url", "One or more URLs to scan (required). You can repeat this flag. Supports http:// and https://.").Required().Strings()
-	webCrawl = webScan.Flag("crawl", "Enable crawling: follow links discovered on the initial page(s).").Default("false").Bool()
-	webDepth = webScan.Flag("depth", "Maximum link depth to follow when crawling. 0 = only the seed URL(s); 1 = seed + direct links; etc.").Default("1").Int()
-	webDelay = webScan.Flag("delay", "Delay (in seconds) between requests to the same domain. Helps respect server load.").Default("1").Int()
+	webScan      = cli.Command("web", "Scan websites for leaked credentials")
+	webUrls      = webScan.Flag("url", "One or more URLs to scan (required). You can repeat this flag. Supports http:// and https://.").Required().Strings()
+	webCrawl     = webScan.Flag("crawl", "Enable crawling: follow links discovered on the initial page(s).").Default("false").Bool()
+	webDepth     = webScan.Flag("depth", "Maximum link depth to follow when crawling. 0 = only the seed URL(s); 1 = seed + direct links; etc.").Default("1").Int()
+	webDelay     = webScan.Flag("delay", "Delay (in seconds) between requests to the same domain. Helps respect server load.").Default("1").Int()
+	webUserAgent = webScan.Flag("user-agent", "User-Agent header sent with each HTTP request. If not set, a descriptive default is used.").String()
 
 	analyzeCmd = analyzer.Command(cli)
 	usingTUI   = false
@@ -1167,11 +1168,17 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics,
 			return scanMetrics, fmt.Errorf("invalid config: you must specify at least one url")
 		}
 
+		if *webUserAgent == "" {
+			ctx.Logger().Info("No user agent set; using default", "user-agent", "trufflehog-web (+https://github.com/trufflesecurity/trufflehog)")
+			*webUserAgent = "trufflehog-web (+https://github.com/trufflesecurity/trufflehog)"
+		}
+
 		cfg := sources.WebConfig{
-			URLs:  *webUrls,
-			Crawl: *webCrawl,
-			Depth: *webDepth,
-			Delay: *webDelay,
+			URLs:      *webUrls,
+			Crawl:     *webCrawl,
+			Depth:     *webDepth,
+			Delay:     *webDelay,
+			UserAgent: *webUserAgent,
 		}
 
 		if ref, err := eng.ScanWeb(ctx, cfg); err != nil {
diff --git a/pkg/pb/sourcespb/sources.pb.go b/pkg/pb/sourcespb/sources.pb.go
index 323b3483b981..4b069b5f533b 100644
--- a/pkg/pb/sourcespb/sources.pb.go
+++ b/pkg/pb/sourcespb/sources.pb.go
@@ -5108,10 +5108,11 @@ type Web struct {
 	sizeCache     protoimpl.SizeCache
 	unknownFields protoimpl.UnknownFields
 
-	Urls  []string `protobuf:"bytes,1,rep,name=urls,proto3" json:"urls,omitempty"`
-	Crawl bool     `protobuf:"varint,2,opt,name=crawl,proto3" json:"crawl,omitempty"`
-	Depth int64    `protobuf:"varint,3,opt,name=depth,proto3" json:"depth,omitempty"`
-	Delay int64    `protobuf:"varint,4,opt,name=delay,proto3" json:"delay,omitempty"`
+	Urls      []string `protobuf:"bytes,1,rep,name=urls,proto3" json:"urls,omitempty"`
+	Crawl     bool     `protobuf:"varint,2,opt,name=crawl,proto3" json:"crawl,omitempty"`
+	Depth     int64    `protobuf:"varint,3,opt,name=depth,proto3" json:"depth,omitempty"`
+	Delay     int64    `protobuf:"varint,4,opt,name=delay,proto3" json:"delay,omitempty"`
+	UserAgent string   `protobuf:"bytes,5,opt,name=user_agent,json=userAgent,proto3" json:"user_agent,omitempty"`
 }
 
 func (x *Web) Reset() {
@@ -5174,6 +5175,13 @@ func (x *Web) GetDelay() int64 {
 	return 0
 }
 
+func (x *Web) GetUserAgent() string {
+	if x != nil {
+		return x.UserAgent
+	}
+	return ""
+}
+
 var File_sources_proto protoreflect.FileDescriptor
 
 var file_sources_proto_rawDesc = []byte{
@@ -5966,108 +5974,110 @@ var file_sources_proto_rawDesc = []byte{
 	0x74, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x70, 0x72, 0x6f, 0x6a,
 	0x65, 0x63, 0x74, 0x49, 0x64, 0x22, 0x26, 0x0a, 0x0e, 0x4a, 0x53, 0x4f, 0x4e, 0x45, 0x6e, 0x75,
 	0x6d, 0x65, 0x72, 0x61, 0x74, 0x6f, 0x72, 0x12, 0x14, 0x0a, 0x05, 0x70, 0x61, 0x74, 0x68, 0x73,
-	0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x05, 0x70, 0x61, 0x74, 0x68, 0x73, 0x22, 0x5b, 0x0a,
+	0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x05, 0x70, 0x61, 0x74, 0x68, 0x73, 0x22, 0x7a, 0x0a,
 	0x03, 0x57, 0x65, 0x62, 0x12, 0x12, 0x0a, 0x04, 0x75, 0x72, 0x6c, 0x73, 0x18, 0x01, 0x20, 0x03,
 	0x28, 0x09, 0x52, 0x04, 0x75, 0x72, 0x6c, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x63, 0x72, 0x61, 0x77,
 	0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, 0x63, 0x72, 0x61, 0x77, 0x6c, 0x12, 0x14,
 	0x0a, 0x05, 0x64, 0x65, 0x70, 0x74, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x05, 0x64,
 	0x65, 0x70, 0x74, 0x68, 0x12, 0x14, 0x0a, 0x05, 0x64, 0x65, 0x6c, 0x61, 0x79, 0x18, 0x04, 0x20,
-	0x01, 0x28, 0x03, 0x52, 0x05, 0x64, 0x65, 0x6c, 0x61, 0x79, 0x2a, 0xd9, 0x09, 0x0a, 0x0a, 0x53,
-	0x6f, 0x75, 0x72, 0x63, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x1d, 0x0a, 0x19, 0x53, 0x4f, 0x55,
-	0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x41, 0x5a, 0x55, 0x52, 0x45, 0x5f, 0x53,
-	0x54, 0x4f, 0x52, 0x41, 0x47, 0x45, 0x10, 0x00, 0x12, 0x19, 0x0a, 0x15, 0x53, 0x4f, 0x55, 0x52,
-	0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x42, 0x49, 0x54, 0x42, 0x55, 0x43, 0x4b, 0x45,
-	0x54, 0x10, 0x01, 0x12, 0x18, 0x0a, 0x14, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59,
-	0x50, 0x45, 0x5f, 0x43, 0x49, 0x52, 0x43, 0x4c, 0x45, 0x43, 0x49, 0x10, 0x02, 0x12, 0x1a, 0x0a,
-	0x16, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x43, 0x4f, 0x4e,
-	0x46, 0x4c, 0x55, 0x45, 0x4e, 0x43, 0x45, 0x10, 0x03, 0x12, 0x16, 0x0a, 0x12, 0x53, 0x4f, 0x55,
-	0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x44, 0x4f, 0x43, 0x4b, 0x45, 0x52, 0x10,
-	0x04, 0x12, 0x13, 0x0a, 0x0f, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45,
-	0x5f, 0x45, 0x43, 0x52, 0x10, 0x05, 0x12, 0x13, 0x0a, 0x0f, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45,
-	0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x43, 0x53, 0x10, 0x06, 0x12, 0x16, 0x0a, 0x12, 0x53,
-	0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x49, 0x54, 0x48, 0x55,
-	0x42, 0x10, 0x07, 0x12, 0x1a, 0x0a, 0x16, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59,
-	0x50, 0x45, 0x5f, 0x50, 0x55, 0x42, 0x4c, 0x49, 0x43, 0x5f, 0x47, 0x49, 0x54, 0x10, 0x08, 0x12,
+	0x01, 0x28, 0x03, 0x52, 0x05, 0x64, 0x65, 0x6c, 0x61, 0x79, 0x12, 0x1d, 0x0a, 0x0a, 0x75, 0x73,
+	0x65, 0x72, 0x5f, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09,
+	0x75, 0x73, 0x65, 0x72, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x2a, 0xd9, 0x09, 0x0a, 0x0a, 0x53, 0x6f,
+	0x75, 0x72, 0x63, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x1d, 0x0a, 0x19, 0x53, 0x4f, 0x55, 0x52,
+	0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x41, 0x5a, 0x55, 0x52, 0x45, 0x5f, 0x53, 0x54,
+	0x4f, 0x52, 0x41, 0x47, 0x45, 0x10, 0x00, 0x12, 0x19, 0x0a, 0x15, 0x53, 0x4f, 0x55, 0x52, 0x43,
+	0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x42, 0x49, 0x54, 0x42, 0x55, 0x43, 0x4b, 0x45, 0x54,
+	0x10, 0x01, 0x12, 0x18, 0x0a, 0x14, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50,
+	0x45, 0x5f, 0x43, 0x49, 0x52, 0x43, 0x4c, 0x45, 0x43, 0x49, 0x10, 0x02, 0x12, 0x1a, 0x0a, 0x16,
+	0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x43, 0x4f, 0x4e, 0x46,
+	0x4c, 0x55, 0x45, 0x4e, 0x43, 0x45, 0x10, 0x03, 0x12, 0x16, 0x0a, 0x12, 0x53, 0x4f, 0x55, 0x52,
+	0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x44, 0x4f, 0x43, 0x4b, 0x45, 0x52, 0x10, 0x04,
+	0x12, 0x13, 0x0a, 0x0f, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f,
+	0x45, 0x43, 0x52, 0x10, 0x05, 0x12, 0x13, 0x0a, 0x0f, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f,
+	0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x43, 0x53, 0x10, 0x06, 0x12, 0x16, 0x0a, 0x12, 0x53, 0x4f,
+	0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x49, 0x54, 0x48, 0x55, 0x42,
+	0x10, 0x07, 0x12, 0x1a, 0x0a, 0x16, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50,
+	0x45, 0x5f, 0x50, 0x55, 0x42, 0x4c, 0x49, 0x43, 0x5f, 0x47, 0x49, 0x54, 0x10, 0x08, 0x12, 0x16,
+	0x0a, 0x12, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x49,
+	0x54, 0x4c, 0x41, 0x42, 0x10, 0x09, 0x12, 0x14, 0x0a, 0x10, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45,
+	0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4a, 0x49, 0x52, 0x41, 0x10, 0x0a, 0x12, 0x24, 0x0a, 0x20,
+	0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4e, 0x50, 0x4d, 0x5f,
+	0x55, 0x4e, 0x41, 0x55, 0x54, 0x48, 0x44, 0x5f, 0x50, 0x41, 0x43, 0x4b, 0x41, 0x47, 0x45, 0x53,
+	0x10, 0x0b, 0x12, 0x25, 0x0a, 0x21, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50,
+	0x45, 0x5f, 0x50, 0x59, 0x50, 0x49, 0x5f, 0x55, 0x4e, 0x41, 0x55, 0x54, 0x48, 0x44, 0x5f, 0x50,
+	0x41, 0x43, 0x4b, 0x41, 0x47, 0x45, 0x53, 0x10, 0x0c, 0x12, 0x12, 0x0a, 0x0e, 0x53, 0x4f, 0x55,
+	0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x33, 0x10, 0x0d, 0x12, 0x15, 0x0a,
+	0x11, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x4c, 0x41,
+	0x43, 0x4b, 0x10, 0x0e, 0x12, 0x1a, 0x0a, 0x16, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54,
+	0x59, 0x50, 0x45, 0x5f, 0x46, 0x49, 0x4c, 0x45, 0x53, 0x59, 0x53, 0x54, 0x45, 0x4d, 0x10, 0x0f,
+	0x12, 0x13, 0x0a, 0x0f, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f,
+	0x47, 0x49, 0x54, 0x10, 0x10, 0x12, 0x14, 0x0a, 0x10, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f,
+	0x54, 0x59, 0x50, 0x45, 0x5f, 0x54, 0x45, 0x53, 0x54, 0x10, 0x11, 0x12, 0x1b, 0x0a, 0x17, 0x53,
+	0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x33, 0x5f, 0x55, 0x4e,
+	0x41, 0x55, 0x54, 0x48, 0x45, 0x44, 0x10, 0x12, 0x12, 0x2a, 0x0a, 0x26, 0x53, 0x4f, 0x55, 0x52,
+	0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x49, 0x54, 0x48, 0x55, 0x42, 0x5f, 0x55,
+	0x4e, 0x41, 0x55, 0x54, 0x48, 0x45, 0x4e, 0x54, 0x49, 0x43, 0x41, 0x54, 0x45, 0x44, 0x5f, 0x4f,
+	0x52, 0x47, 0x10, 0x13, 0x12, 0x19, 0x0a, 0x15, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54,
+	0x59, 0x50, 0x45, 0x5f, 0x42, 0x55, 0x49, 0x4c, 0x44, 0x4b, 0x49, 0x54, 0x45, 0x10, 0x14, 0x12,
 	0x16, 0x0a, 0x12, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47,
-	0x49, 0x54, 0x4c, 0x41, 0x42, 0x10, 0x09, 0x12, 0x14, 0x0a, 0x10, 0x53, 0x4f, 0x55, 0x52, 0x43,
-	0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4a, 0x49, 0x52, 0x41, 0x10, 0x0a, 0x12, 0x24, 0x0a,
-	0x20, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4e, 0x50, 0x4d,
-	0x5f, 0x55, 0x4e, 0x41, 0x55, 0x54, 0x48, 0x44, 0x5f, 0x50, 0x41, 0x43, 0x4b, 0x41, 0x47, 0x45,
-	0x53, 0x10, 0x0b, 0x12, 0x25, 0x0a, 0x21, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59,
-	0x50, 0x45, 0x5f, 0x50, 0x59, 0x50, 0x49, 0x5f, 0x55, 0x4e, 0x41, 0x55, 0x54, 0x48, 0x44, 0x5f,
-	0x50, 0x41, 0x43, 0x4b, 0x41, 0x47, 0x45, 0x53, 0x10, 0x0c, 0x12, 0x12, 0x0a, 0x0e, 0x53, 0x4f,
-	0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x33, 0x10, 0x0d, 0x12, 0x15,
-	0x0a, 0x11, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x4c,
-	0x41, 0x43, 0x4b, 0x10, 0x0e, 0x12, 0x1a, 0x0a, 0x16, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f,
-	0x54, 0x59, 0x50, 0x45, 0x5f, 0x46, 0x49, 0x4c, 0x45, 0x53, 0x59, 0x53, 0x54, 0x45, 0x4d, 0x10,
-	0x0f, 0x12, 0x13, 0x0a, 0x0f, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45,
-	0x5f, 0x47, 0x49, 0x54, 0x10, 0x10, 0x12, 0x14, 0x0a, 0x10, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45,
-	0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x54, 0x45, 0x53, 0x54, 0x10, 0x11, 0x12, 0x1b, 0x0a, 0x17,
-	0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x33, 0x5f, 0x55,
-	0x4e, 0x41, 0x55, 0x54, 0x48, 0x45, 0x44, 0x10, 0x12, 0x12, 0x2a, 0x0a, 0x26, 0x53, 0x4f, 0x55,
-	0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x49, 0x54, 0x48, 0x55, 0x42, 0x5f,
-	0x55, 0x4e, 0x41, 0x55, 0x54, 0x48, 0x45, 0x4e, 0x54, 0x49, 0x43, 0x41, 0x54, 0x45, 0x44, 0x5f,
-	0x4f, 0x52, 0x47, 0x10, 0x13, 0x12, 0x19, 0x0a, 0x15, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f,
-	0x54, 0x59, 0x50, 0x45, 0x5f, 0x42, 0x55, 0x49, 0x4c, 0x44, 0x4b, 0x49, 0x54, 0x45, 0x10, 0x14,
-	0x12, 0x16, 0x0a, 0x12, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f,
-	0x47, 0x45, 0x52, 0x52, 0x49, 0x54, 0x10, 0x15, 0x12, 0x17, 0x0a, 0x13, 0x53, 0x4f, 0x55, 0x52,
-	0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4a, 0x45, 0x4e, 0x4b, 0x49, 0x4e, 0x53, 0x10,
-	0x16, 0x12, 0x15, 0x0a, 0x11, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45,
-	0x5f, 0x54, 0x45, 0x41, 0x4d, 0x53, 0x10, 0x17, 0x12, 0x21, 0x0a, 0x1d, 0x53, 0x4f, 0x55, 0x52,
-	0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4a, 0x46, 0x52, 0x4f, 0x47, 0x5f, 0x41, 0x52,
-	0x54, 0x49, 0x46, 0x41, 0x43, 0x54, 0x4f, 0x52, 0x59, 0x10, 0x18, 0x12, 0x16, 0x0a, 0x12, 0x53,
-	0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x59, 0x53, 0x4c, 0x4f,
-	0x47, 0x10, 0x19, 0x12, 0x27, 0x0a, 0x23, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59,
-	0x50, 0x45, 0x5f, 0x50, 0x55, 0x42, 0x4c, 0x49, 0x43, 0x5f, 0x45, 0x56, 0x45, 0x4e, 0x54, 0x5f,
-	0x4d, 0x4f, 0x4e, 0x49, 0x54, 0x4f, 0x52, 0x49, 0x4e, 0x47, 0x10, 0x1a, 0x12, 0x1e, 0x0a, 0x1a,
-	0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x4c, 0x41, 0x43,
-	0x4b, 0x5f, 0x52, 0x45, 0x41, 0x4c, 0x54, 0x49, 0x4d, 0x45, 0x10, 0x1b, 0x12, 0x1c, 0x0a, 0x18,
-	0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x4f, 0x4f, 0x47,
-	0x4c, 0x45, 0x5f, 0x44, 0x52, 0x49, 0x56, 0x45, 0x10, 0x1c, 0x12, 0x1a, 0x0a, 0x16, 0x53, 0x4f,
-	0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x48, 0x41, 0x52, 0x45, 0x50,
-	0x4f, 0x49, 0x4e, 0x54, 0x10, 0x1d, 0x12, 0x1c, 0x0a, 0x18, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45,
-	0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x43, 0x53, 0x5f, 0x55, 0x4e, 0x41, 0x55, 0x54, 0x48,
-	0x45, 0x44, 0x10, 0x1e, 0x12, 0x1b, 0x0a, 0x17, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54,
-	0x59, 0x50, 0x45, 0x5f, 0x41, 0x5a, 0x55, 0x52, 0x45, 0x5f, 0x52, 0x45, 0x50, 0x4f, 0x53, 0x10,
-	0x1f, 0x12, 0x18, 0x0a, 0x14, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45,
-	0x5f, 0x54, 0x52, 0x41, 0x56, 0x49, 0x53, 0x43, 0x49, 0x10, 0x20, 0x12, 0x17, 0x0a, 0x13, 0x53,
-	0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x50, 0x4f, 0x53, 0x54, 0x4d,
-	0x41, 0x4e, 0x10, 0x21, 0x12, 0x17, 0x0a, 0x13, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54,
-	0x59, 0x50, 0x45, 0x5f, 0x57, 0x45, 0x42, 0x48, 0x4f, 0x4f, 0x4b, 0x10, 0x22, 0x12, 0x1d, 0x0a,
-	0x19, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x45, 0x4c, 0x41,
-	0x53, 0x54, 0x49, 0x43, 0x53, 0x45, 0x41, 0x52, 0x43, 0x48, 0x10, 0x23, 0x12, 0x1b, 0x0a, 0x17,
-	0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x48, 0x55, 0x47, 0x47,
-	0x49, 0x4e, 0x47, 0x46, 0x41, 0x43, 0x45, 0x10, 0x24, 0x12, 0x23, 0x0a, 0x1f, 0x53, 0x4f, 0x55,
-	0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x49, 0x54, 0x48, 0x55, 0x42, 0x5f,
-	0x45, 0x58, 0x50, 0x45, 0x52, 0x49, 0x4d, 0x45, 0x4e, 0x54, 0x41, 0x4c, 0x10, 0x25, 0x12, 0x16,
-	0x0a, 0x12, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x45,
-	0x4e, 0x54, 0x52, 0x59, 0x10, 0x26, 0x12, 0x1f, 0x0a, 0x1b, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45,
-	0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x49, 0x54, 0x48, 0x55, 0x42, 0x5f, 0x52, 0x45, 0x41,
-	0x4c, 0x54, 0x49, 0x4d, 0x45, 0x10, 0x27, 0x12, 0x15, 0x0a, 0x11, 0x53, 0x4f, 0x55, 0x52, 0x43,
-	0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x54, 0x44, 0x49, 0x4e, 0x10, 0x28, 0x12, 0x20,
-	0x0a, 0x1c, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x4c,
-	0x41, 0x43, 0x4b, 0x5f, 0x43, 0x4f, 0x4e, 0x54, 0x49, 0x4e, 0x55, 0x4f, 0x55, 0x53, 0x10, 0x29,
-	0x12, 0x1f, 0x0a, 0x1b, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f,
-	0x4a, 0x53, 0x4f, 0x4e, 0x5f, 0x45, 0x4e, 0x55, 0x4d, 0x45, 0x52, 0x41, 0x54, 0x4f, 0x52, 0x10,
-	0x2a, 0x12, 0x13, 0x0a, 0x0f, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45,
-	0x5f, 0x57, 0x45, 0x42, 0x10, 0x2b, 0x2a, 0x47, 0x0a, 0x19, 0x42, 0x69, 0x74, 0x62, 0x75, 0x63,
-	0x6b, 0x65, 0x74, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x54,
-	0x79, 0x70, 0x65, 0x12, 0x0e, 0x0a, 0x0a, 0x41, 0x55, 0x54, 0x4f, 0x44, 0x45, 0x54, 0x45, 0x43,
-	0x54, 0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, 0x43, 0x4c, 0x4f, 0x55, 0x44, 0x10, 0x01, 0x12, 0x0f,
-	0x0a, 0x0b, 0x44, 0x41, 0x54, 0x41, 0x5f, 0x43, 0x45, 0x4e, 0x54, 0x45, 0x52, 0x10, 0x02, 0x2a,
-	0x87, 0x01, 0x0a, 0x14, 0x4a, 0x69, 0x72, 0x61, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x61,
-	0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x12, 0x25, 0x0a, 0x21, 0x4a, 0x49, 0x52, 0x41,
-	0x5f, 0x49, 0x4e, 0x53, 0x54, 0x41, 0x4c, 0x4c, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x54, 0x59,
-	0x50, 0x45, 0x5f, 0x41, 0x55, 0x54, 0x4f, 0x44, 0x45, 0x54, 0x45, 0x43, 0x54, 0x10, 0x00, 0x12,
-	0x20, 0x0a, 0x1c, 0x4a, 0x49, 0x52, 0x41, 0x5f, 0x49, 0x4e, 0x53, 0x54, 0x41, 0x4c, 0x4c, 0x41,
-	0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x43, 0x4c, 0x4f, 0x55, 0x44, 0x10,
-	0x01, 0x12, 0x26, 0x0a, 0x22, 0x4a, 0x49, 0x52, 0x41, 0x5f, 0x49, 0x4e, 0x53, 0x54, 0x41, 0x4c,
-	0x4c, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x44, 0x41, 0x54, 0x41,
-	0x5f, 0x43, 0x45, 0x4e, 0x54, 0x45, 0x52, 0x10, 0x02, 0x42, 0x3b, 0x5a, 0x39, 0x67, 0x69, 0x74,
-	0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73,
-	0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68,
-	0x6f, 0x67, 0x2f, 0x76, 0x33, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x73, 0x6f, 0x75,
-	0x72, 0x63, 0x65, 0x73, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
+	0x45, 0x52, 0x52, 0x49, 0x54, 0x10, 0x15, 0x12, 0x17, 0x0a, 0x13, 0x53, 0x4f, 0x55, 0x52, 0x43,
+	0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4a, 0x45, 0x4e, 0x4b, 0x49, 0x4e, 0x53, 0x10, 0x16,
+	0x12, 0x15, 0x0a, 0x11, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f,
+	0x54, 0x45, 0x41, 0x4d, 0x53, 0x10, 0x17, 0x12, 0x21, 0x0a, 0x1d, 0x53, 0x4f, 0x55, 0x52, 0x43,
+	0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4a, 0x46, 0x52, 0x4f, 0x47, 0x5f, 0x41, 0x52, 0x54,
+	0x49, 0x46, 0x41, 0x43, 0x54, 0x4f, 0x52, 0x59, 0x10, 0x18, 0x12, 0x16, 0x0a, 0x12, 0x53, 0x4f,
+	0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x59, 0x53, 0x4c, 0x4f, 0x47,
+	0x10, 0x19, 0x12, 0x27, 0x0a, 0x23, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50,
+	0x45, 0x5f, 0x50, 0x55, 0x42, 0x4c, 0x49, 0x43, 0x5f, 0x45, 0x56, 0x45, 0x4e, 0x54, 0x5f, 0x4d,
+	0x4f, 0x4e, 0x49, 0x54, 0x4f, 0x52, 0x49, 0x4e, 0x47, 0x10, 0x1a, 0x12, 0x1e, 0x0a, 0x1a, 0x53,
+	0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x4c, 0x41, 0x43, 0x4b,
+	0x5f, 0x52, 0x45, 0x41, 0x4c, 0x54, 0x49, 0x4d, 0x45, 0x10, 0x1b, 0x12, 0x1c, 0x0a, 0x18, 0x53,
+	0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x4f, 0x4f, 0x47, 0x4c,
+	0x45, 0x5f, 0x44, 0x52, 0x49, 0x56, 0x45, 0x10, 0x1c, 0x12, 0x1a, 0x0a, 0x16, 0x53, 0x4f, 0x55,
+	0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x48, 0x41, 0x52, 0x45, 0x50, 0x4f,
+	0x49, 0x4e, 0x54, 0x10, 0x1d, 0x12, 0x1c, 0x0a, 0x18, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f,
+	0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x43, 0x53, 0x5f, 0x55, 0x4e, 0x41, 0x55, 0x54, 0x48, 0x45,
+	0x44, 0x10, 0x1e, 0x12, 0x1b, 0x0a, 0x17, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59,
+	0x50, 0x45, 0x5f, 0x41, 0x5a, 0x55, 0x52, 0x45, 0x5f, 0x52, 0x45, 0x50, 0x4f, 0x53, 0x10, 0x1f,
+	0x12, 0x18, 0x0a, 0x14, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f,
+	0x54, 0x52, 0x41, 0x56, 0x49, 0x53, 0x43, 0x49, 0x10, 0x20, 0x12, 0x17, 0x0a, 0x13, 0x53, 0x4f,
+	0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x50, 0x4f, 0x53, 0x54, 0x4d, 0x41,
+	0x4e, 0x10, 0x21, 0x12, 0x17, 0x0a, 0x13, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59,
+	0x50, 0x45, 0x5f, 0x57, 0x45, 0x42, 0x48, 0x4f, 0x4f, 0x4b, 0x10, 0x22, 0x12, 0x1d, 0x0a, 0x19,
+	0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x45, 0x4c, 0x41, 0x53,
+	0x54, 0x49, 0x43, 0x53, 0x45, 0x41, 0x52, 0x43, 0x48, 0x10, 0x23, 0x12, 0x1b, 0x0a, 0x17, 0x53,
+	0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x48, 0x55, 0x47, 0x47, 0x49,
+	0x4e, 0x47, 0x46, 0x41, 0x43, 0x45, 0x10, 0x24, 0x12, 0x23, 0x0a, 0x1f, 0x53, 0x4f, 0x55, 0x52,
+	0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x49, 0x54, 0x48, 0x55, 0x42, 0x5f, 0x45,
+	0x58, 0x50, 0x45, 0x52, 0x49, 0x4d, 0x45, 0x4e, 0x54, 0x41, 0x4c, 0x10, 0x25, 0x12, 0x16, 0x0a,
+	0x12, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x45, 0x4e,
+	0x54, 0x52, 0x59, 0x10, 0x26, 0x12, 0x1f, 0x0a, 0x1b, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f,
+	0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x49, 0x54, 0x48, 0x55, 0x42, 0x5f, 0x52, 0x45, 0x41, 0x4c,
+	0x54, 0x49, 0x4d, 0x45, 0x10, 0x27, 0x12, 0x15, 0x0a, 0x11, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45,
+	0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x54, 0x44, 0x49, 0x4e, 0x10, 0x28, 0x12, 0x20, 0x0a,
+	0x1c, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x4c, 0x41,
+	0x43, 0x4b, 0x5f, 0x43, 0x4f, 0x4e, 0x54, 0x49, 0x4e, 0x55, 0x4f, 0x55, 0x53, 0x10, 0x29, 0x12,
+	0x1f, 0x0a, 0x1b, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4a,
+	0x53, 0x4f, 0x4e, 0x5f, 0x45, 0x4e, 0x55, 0x4d, 0x45, 0x52, 0x41, 0x54, 0x4f, 0x52, 0x10, 0x2a,
+	0x12, 0x13, 0x0a, 0x0f, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f,
+	0x57, 0x45, 0x42, 0x10, 0x2b, 0x2a, 0x47, 0x0a, 0x19, 0x42, 0x69, 0x74, 0x62, 0x75, 0x63, 0x6b,
+	0x65, 0x74, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79,
+	0x70, 0x65, 0x12, 0x0e, 0x0a, 0x0a, 0x41, 0x55, 0x54, 0x4f, 0x44, 0x45, 0x54, 0x45, 0x43, 0x54,
+	0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, 0x43, 0x4c, 0x4f, 0x55, 0x44, 0x10, 0x01, 0x12, 0x0f, 0x0a,
+	0x0b, 0x44, 0x41, 0x54, 0x41, 0x5f, 0x43, 0x45, 0x4e, 0x54, 0x45, 0x52, 0x10, 0x02, 0x2a, 0x87,
+	0x01, 0x0a, 0x14, 0x4a, 0x69, 0x72, 0x61, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x61, 0x74,
+	0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x12, 0x25, 0x0a, 0x21, 0x4a, 0x49, 0x52, 0x41, 0x5f,
+	0x49, 0x4e, 0x53, 0x54, 0x41, 0x4c, 0x4c, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x54, 0x59, 0x50,
+	0x45, 0x5f, 0x41, 0x55, 0x54, 0x4f, 0x44, 0x45, 0x54, 0x45, 0x43, 0x54, 0x10, 0x00, 0x12, 0x20,
+	0x0a, 0x1c, 0x4a, 0x49, 0x52, 0x41, 0x5f, 0x49, 0x4e, 0x53, 0x54, 0x41, 0x4c, 0x4c, 0x41, 0x54,
+	0x49, 0x4f, 0x4e, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x43, 0x4c, 0x4f, 0x55, 0x44, 0x10, 0x01,
+	0x12, 0x26, 0x0a, 0x22, 0x4a, 0x49, 0x52, 0x41, 0x5f, 0x49, 0x4e, 0x53, 0x54, 0x41, 0x4c, 0x4c,
+	0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x44, 0x41, 0x54, 0x41, 0x5f,
+	0x43, 0x45, 0x4e, 0x54, 0x45, 0x52, 0x10, 0x02, 0x42, 0x3b, 0x5a, 0x39, 0x67, 0x69, 0x74, 0x68,
+	0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73, 0x65,
+	0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68, 0x6f,
+	0x67, 0x2f, 0x76, 0x33, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x73, 0x6f, 0x75, 0x72,
+	0x63, 0x65, 0x73, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
 }
 
 var (
diff --git a/pkg/pb/sourcespb/sources.pb.validate.go b/pkg/pb/sourcespb/sources.pb.validate.go
index d64382b13e87..823472b035bb 100644
--- a/pkg/pb/sourcespb/sources.pb.validate.go
+++ b/pkg/pb/sourcespb/sources.pb.validate.go
@@ -7266,6 +7266,8 @@ func (m *Web) validate(all bool) error {
 
 	// no validation rules for Delay
 
+	// no validation rules for UserAgent
+
 	if len(errors) > 0 {
 		return WebMultiError(errors)
 	}
diff --git a/pkg/sources/sources.go b/pkg/sources/sources.go
index 163c8681390e..a5cb54129bd8 100644
--- a/pkg/sources/sources.go
+++ b/pkg/sources/sources.go
@@ -485,17 +485,18 @@ type JSONEnumeratorConfig struct {
 // WebConfig defines the configuration for the web source.
 type WebConfig struct {
 	// URL is the list of starting points for scanning.
-	URLs []string `json:"url" mapstructure:"url"`
+	URLs []string
 
 	// Crawl determines whether to follow links from the starting page.
-	Crawl bool `json:"crawl" mapstructure:"crawl"`
+	Crawl bool
 
 	// Depth controls how many link hops to follow when Crawl is true.
 	// 0 = only the starting URL, 1 = starting URL and direct links, etc.
-	Depth int `json:"depth" mapstructure:"depth"`
+	Depth int
 
 	// Delay is the delay (in seconds) between requests to the same domain.
-	Delay int `json:"delay" mapstructure:"delay"`
+	Delay     int
+	UserAgent string
 }
 
 // Progress is used to update job completion progress across sources.
diff --git a/pkg/sources/web/web.go b/pkg/sources/web/web.go
index fb7fc5d916ac..d29daa285809 100644
--- a/pkg/sources/web/web.go
+++ b/pkg/sources/web/web.go
@@ -53,6 +53,7 @@ func (s *Source) Init(ctx context.Context, name string, jobId sources.JobID, sou
 	s.verify = verify
 	s.concurrency = concurrency
 	// If s.concurrency is 0, use 1
+	// TODO: make it configurable
 	if s.concurrency == 0 {
 		s.concurrency = 1
 	}
@@ -61,6 +62,11 @@ func (s *Source) Init(ctx context.Context, name string, jobId sources.JobID, sou
 		return fmt.Errorf("error unmarshalling connection: %w", err)
 	}
 
+	// Use the user-provided User-Agent if set; otherwise fall back to a default that identifies TruffleHog.
+	if s.conn.GetUserAgent() == "" {
+		s.conn.UserAgent = "trufflehog-web (+https://github.com/trufflesecurity/trufflehog)"
+	}
+
 	// validations
 	if len(s.conn.GetUrls()) == 0 {
 		return errors.New("no URL provided")
@@ -112,7 +118,7 @@ func (s *Source) crawlURL(ctx context.Context, seedURL string, chunksChan chan *
 	}
 
 	collector := colly.NewCollector(
-		colly.UserAgent("trufflehog-web (+https://github.com/trufflesecurity/trufflehog)"),
+		colly.UserAgent(s.conn.GetUserAgent()),
 		colly.AllowedDomains(url.Hostname(), fmt.Sprintf("*.%s", url.Hostname())), // with subdomains
 		colly.Async(true),
 	)
diff --git a/proto/sources.proto b/proto/sources.proto
index 8b94a629a572..1712150eba58 100644
--- a/proto/sources.proto
+++ b/proto/sources.proto
@@ -561,4 +561,5 @@ message Web {
   bool crawl = 2;
   int64 depth = 3;
   int64 delay = 4;
+  string user_agent = 5;
 }

From 8f5a5fc77643624105f86f4013f771ebe88fc86a Mon Sep 17 00:00:00 2001
From: Kashif Khan <kashif.khan@trufflesec.com>
Date: Mon, 30 Mar 2026 13:15:13 +0500
Subject: [PATCH 06/20] made ignore-robots configurable

---
 main.go                                 |  24 +--
 pkg/pb/sourcespb/sources.pb.go          | 229 +++++++++++++-----------
 pkg/pb/sourcespb/sources.pb.validate.go |   2 +
 pkg/sources/sources.go                  |   5 +-
 pkg/sources/web/web.go                  |  25 ++-
 proto/sources.proto                     |   1 +
 6 files changed, 157 insertions(+), 129 deletions(-)

diff --git a/main.go b/main.go
index 47189dc73630..2cd682b36fde 100644
--- a/main.go
+++ b/main.go
@@ -276,12 +276,13 @@ var (
 	jsonEnumeratorScan  = cli.Command("json-enumerator", "Find credentials from a JSON enumerator input.")
 	jsonEnumeratorPaths = jsonEnumeratorScan.Arg("path", "Path to JSON enumerator file to scan.").Strings()
 
-	webScan      = cli.Command("web", "Scan websites for leaked credentials")
-	webUrls      = webScan.Flag("url", "One or more URLs to scan (required). You can repeat this flag. Supports http:// and https://.").Required().Strings()
-	webCrawl     = webScan.Flag("crawl", "Enable crawling: follow links discovered on the initial page(s).").Default("false").Bool()
-	webDepth     = webScan.Flag("depth", "Maximum link depth to follow when crawling. 0 = only the seed URL(s); 1 = seed + direct links; etc.").Default("1").Int()
-	webDelay     = webScan.Flag("delay", "Delay (in seconds) between requests to the same domain. Helps respect server load.").Default("1").Int()
-	webUserAgent = webScan.Flag("user-agent", "User-Agent header sent with each HTTP request. If not set, a descriptive default is used.").String()
+	webScan         = cli.Command("web", "Scan websites for leaked credentials")
+	webUrls         = webScan.Flag("url", "One or more URLs to scan (required). You can repeat this flag. Supports http:// and https://.").Required().Strings()
+	webCrawl        = webScan.Flag("crawl", "Enable crawling: follow links discovered on the initial page(s).").Default("false").Bool()
+	webDepth        = webScan.Flag("depth", "Maximum link depth to follow when crawling. 0 = only the seed URL(s); 1 = seed + direct links; etc.").Default("1").Int()
+	webDelay        = webScan.Flag("delay", "Delay (in seconds) between requests to the same domain. Helps respect server load.").Default("1").Int()
+	webUserAgent    = webScan.Flag("user-agent", "User-Agent header sent with each HTTP request. If not set, a descriptive default is used.").String()
+	webIgnoreRobots = webScan.Flag("ignore-robots", "Ignore robots.txt rules. Use only if you have permission to crawl the site, otherwise this may violate the site's policies.").Default("false").Bool()
 
 	analyzeCmd = analyzer.Command(cli)
 	usingTUI   = false
@@ -1174,11 +1175,12 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics,
 		}
 
 		cfg := sources.WebConfig{
-			URLs:      *webUrls,
-			Crawl:     *webCrawl,
-			Depth:     *webDepth,
-			Delay:     *webDelay,
-			UserAgent: *webUserAgent,
+			URLs:         *webUrls,
+			Crawl:        *webCrawl,
+			Depth:        *webDepth,
+			Delay:        *webDelay,
+			UserAgent:    *webUserAgent,
+			IgnoreRobots: *webIgnoreRobots,
 		}
 
 		if ref, err := eng.ScanWeb(ctx, cfg); err != nil {
diff --git a/pkg/pb/sourcespb/sources.pb.go b/pkg/pb/sourcespb/sources.pb.go
index 4b069b5f533b..08fd94b6a066 100644
--- a/pkg/pb/sourcespb/sources.pb.go
+++ b/pkg/pb/sourcespb/sources.pb.go
@@ -5108,11 +5108,12 @@ type Web struct {
 	sizeCache     protoimpl.SizeCache
 	unknownFields protoimpl.UnknownFields
 
-	Urls      []string `protobuf:"bytes,1,rep,name=urls,proto3" json:"urls,omitempty"`
-	Crawl     bool     `protobuf:"varint,2,opt,name=crawl,proto3" json:"crawl,omitempty"`
-	Depth     int64    `protobuf:"varint,3,opt,name=depth,proto3" json:"depth,omitempty"`
-	Delay     int64    `protobuf:"varint,4,opt,name=delay,proto3" json:"delay,omitempty"`
-	UserAgent string   `protobuf:"bytes,5,opt,name=user_agent,json=userAgent,proto3" json:"user_agent,omitempty"`
+	Urls         []string `protobuf:"bytes,1,rep,name=urls,proto3" json:"urls,omitempty"`
+	Crawl        bool     `protobuf:"varint,2,opt,name=crawl,proto3" json:"crawl,omitempty"`
+	Depth        int64    `protobuf:"varint,3,opt,name=depth,proto3" json:"depth,omitempty"`
+	Delay        int64    `protobuf:"varint,4,opt,name=delay,proto3" json:"delay,omitempty"`
+	UserAgent    string   `protobuf:"bytes,5,opt,name=user_agent,json=userAgent,proto3" json:"user_agent,omitempty"`
+	IgnoreRobots bool     `protobuf:"varint,6,opt,name=ignore_robots,json=ignoreRobots,proto3" json:"ignore_robots,omitempty"`
 }
 
 func (x *Web) Reset() {
@@ -5182,6 +5183,13 @@ func (x *Web) GetUserAgent() string {
 	return ""
 }
 
+func (x *Web) GetIgnoreRobots() bool {
+	if x != nil {
+		return x.IgnoreRobots
+	}
+	return false
+}
+
 var File_sources_proto protoreflect.FileDescriptor
 
 var file_sources_proto_rawDesc = []byte{
@@ -5974,110 +5982,113 @@ var file_sources_proto_rawDesc = []byte{
 	0x74, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x70, 0x72, 0x6f, 0x6a,
 	0x65, 0x63, 0x74, 0x49, 0x64, 0x22, 0x26, 0x0a, 0x0e, 0x4a, 0x53, 0x4f, 0x4e, 0x45, 0x6e, 0x75,
 	0x6d, 0x65, 0x72, 0x61, 0x74, 0x6f, 0x72, 0x12, 0x14, 0x0a, 0x05, 0x70, 0x61, 0x74, 0x68, 0x73,
-	0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x05, 0x70, 0x61, 0x74, 0x68, 0x73, 0x22, 0x7a, 0x0a,
-	0x03, 0x57, 0x65, 0x62, 0x12, 0x12, 0x0a, 0x04, 0x75, 0x72, 0x6c, 0x73, 0x18, 0x01, 0x20, 0x03,
-	0x28, 0x09, 0x52, 0x04, 0x75, 0x72, 0x6c, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x63, 0x72, 0x61, 0x77,
-	0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, 0x63, 0x72, 0x61, 0x77, 0x6c, 0x12, 0x14,
-	0x0a, 0x05, 0x64, 0x65, 0x70, 0x74, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x05, 0x64,
-	0x65, 0x70, 0x74, 0x68, 0x12, 0x14, 0x0a, 0x05, 0x64, 0x65, 0x6c, 0x61, 0x79, 0x18, 0x04, 0x20,
-	0x01, 0x28, 0x03, 0x52, 0x05, 0x64, 0x65, 0x6c, 0x61, 0x79, 0x12, 0x1d, 0x0a, 0x0a, 0x75, 0x73,
-	0x65, 0x72, 0x5f, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09,
-	0x75, 0x73, 0x65, 0x72, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x2a, 0xd9, 0x09, 0x0a, 0x0a, 0x53, 0x6f,
-	0x75, 0x72, 0x63, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x1d, 0x0a, 0x19, 0x53, 0x4f, 0x55, 0x52,
-	0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x41, 0x5a, 0x55, 0x52, 0x45, 0x5f, 0x53, 0x54,
-	0x4f, 0x52, 0x41, 0x47, 0x45, 0x10, 0x00, 0x12, 0x19, 0x0a, 0x15, 0x53, 0x4f, 0x55, 0x52, 0x43,
-	0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x42, 0x49, 0x54, 0x42, 0x55, 0x43, 0x4b, 0x45, 0x54,
-	0x10, 0x01, 0x12, 0x18, 0x0a, 0x14, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50,
-	0x45, 0x5f, 0x43, 0x49, 0x52, 0x43, 0x4c, 0x45, 0x43, 0x49, 0x10, 0x02, 0x12, 0x1a, 0x0a, 0x16,
-	0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x43, 0x4f, 0x4e, 0x46,
-	0x4c, 0x55, 0x45, 0x4e, 0x43, 0x45, 0x10, 0x03, 0x12, 0x16, 0x0a, 0x12, 0x53, 0x4f, 0x55, 0x52,
-	0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x44, 0x4f, 0x43, 0x4b, 0x45, 0x52, 0x10, 0x04,
-	0x12, 0x13, 0x0a, 0x0f, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f,
-	0x45, 0x43, 0x52, 0x10, 0x05, 0x12, 0x13, 0x0a, 0x0f, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f,
-	0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x43, 0x53, 0x10, 0x06, 0x12, 0x16, 0x0a, 0x12, 0x53, 0x4f,
-	0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x49, 0x54, 0x48, 0x55, 0x42,
-	0x10, 0x07, 0x12, 0x1a, 0x0a, 0x16, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50,
-	0x45, 0x5f, 0x50, 0x55, 0x42, 0x4c, 0x49, 0x43, 0x5f, 0x47, 0x49, 0x54, 0x10, 0x08, 0x12, 0x16,
-	0x0a, 0x12, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x49,
-	0x54, 0x4c, 0x41, 0x42, 0x10, 0x09, 0x12, 0x14, 0x0a, 0x10, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45,
-	0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4a, 0x49, 0x52, 0x41, 0x10, 0x0a, 0x12, 0x24, 0x0a, 0x20,
-	0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4e, 0x50, 0x4d, 0x5f,
-	0x55, 0x4e, 0x41, 0x55, 0x54, 0x48, 0x44, 0x5f, 0x50, 0x41, 0x43, 0x4b, 0x41, 0x47, 0x45, 0x53,
-	0x10, 0x0b, 0x12, 0x25, 0x0a, 0x21, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50,
-	0x45, 0x5f, 0x50, 0x59, 0x50, 0x49, 0x5f, 0x55, 0x4e, 0x41, 0x55, 0x54, 0x48, 0x44, 0x5f, 0x50,
-	0x41, 0x43, 0x4b, 0x41, 0x47, 0x45, 0x53, 0x10, 0x0c, 0x12, 0x12, 0x0a, 0x0e, 0x53, 0x4f, 0x55,
-	0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x33, 0x10, 0x0d, 0x12, 0x15, 0x0a,
-	0x11, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x4c, 0x41,
-	0x43, 0x4b, 0x10, 0x0e, 0x12, 0x1a, 0x0a, 0x16, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54,
-	0x59, 0x50, 0x45, 0x5f, 0x46, 0x49, 0x4c, 0x45, 0x53, 0x59, 0x53, 0x54, 0x45, 0x4d, 0x10, 0x0f,
-	0x12, 0x13, 0x0a, 0x0f, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f,
-	0x47, 0x49, 0x54, 0x10, 0x10, 0x12, 0x14, 0x0a, 0x10, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f,
-	0x54, 0x59, 0x50, 0x45, 0x5f, 0x54, 0x45, 0x53, 0x54, 0x10, 0x11, 0x12, 0x1b, 0x0a, 0x17, 0x53,
-	0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x33, 0x5f, 0x55, 0x4e,
-	0x41, 0x55, 0x54, 0x48, 0x45, 0x44, 0x10, 0x12, 0x12, 0x2a, 0x0a, 0x26, 0x53, 0x4f, 0x55, 0x52,
-	0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x49, 0x54, 0x48, 0x55, 0x42, 0x5f, 0x55,
-	0x4e, 0x41, 0x55, 0x54, 0x48, 0x45, 0x4e, 0x54, 0x49, 0x43, 0x41, 0x54, 0x45, 0x44, 0x5f, 0x4f,
-	0x52, 0x47, 0x10, 0x13, 0x12, 0x19, 0x0a, 0x15, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54,
-	0x59, 0x50, 0x45, 0x5f, 0x42, 0x55, 0x49, 0x4c, 0x44, 0x4b, 0x49, 0x54, 0x45, 0x10, 0x14, 0x12,
-	0x16, 0x0a, 0x12, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47,
-	0x45, 0x52, 0x52, 0x49, 0x54, 0x10, 0x15, 0x12, 0x17, 0x0a, 0x13, 0x53, 0x4f, 0x55, 0x52, 0x43,
-	0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4a, 0x45, 0x4e, 0x4b, 0x49, 0x4e, 0x53, 0x10, 0x16,
-	0x12, 0x15, 0x0a, 0x11, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f,
-	0x54, 0x45, 0x41, 0x4d, 0x53, 0x10, 0x17, 0x12, 0x21, 0x0a, 0x1d, 0x53, 0x4f, 0x55, 0x52, 0x43,
-	0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4a, 0x46, 0x52, 0x4f, 0x47, 0x5f, 0x41, 0x52, 0x54,
-	0x49, 0x46, 0x41, 0x43, 0x54, 0x4f, 0x52, 0x59, 0x10, 0x18, 0x12, 0x16, 0x0a, 0x12, 0x53, 0x4f,
-	0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x59, 0x53, 0x4c, 0x4f, 0x47,
-	0x10, 0x19, 0x12, 0x27, 0x0a, 0x23, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50,
-	0x45, 0x5f, 0x50, 0x55, 0x42, 0x4c, 0x49, 0x43, 0x5f, 0x45, 0x56, 0x45, 0x4e, 0x54, 0x5f, 0x4d,
-	0x4f, 0x4e, 0x49, 0x54, 0x4f, 0x52, 0x49, 0x4e, 0x47, 0x10, 0x1a, 0x12, 0x1e, 0x0a, 0x1a, 0x53,
-	0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x4c, 0x41, 0x43, 0x4b,
-	0x5f, 0x52, 0x45, 0x41, 0x4c, 0x54, 0x49, 0x4d, 0x45, 0x10, 0x1b, 0x12, 0x1c, 0x0a, 0x18, 0x53,
-	0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x4f, 0x4f, 0x47, 0x4c,
-	0x45, 0x5f, 0x44, 0x52, 0x49, 0x56, 0x45, 0x10, 0x1c, 0x12, 0x1a, 0x0a, 0x16, 0x53, 0x4f, 0x55,
-	0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x48, 0x41, 0x52, 0x45, 0x50, 0x4f,
-	0x49, 0x4e, 0x54, 0x10, 0x1d, 0x12, 0x1c, 0x0a, 0x18, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f,
-	0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x43, 0x53, 0x5f, 0x55, 0x4e, 0x41, 0x55, 0x54, 0x48, 0x45,
-	0x44, 0x10, 0x1e, 0x12, 0x1b, 0x0a, 0x17, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59,
-	0x50, 0x45, 0x5f, 0x41, 0x5a, 0x55, 0x52, 0x45, 0x5f, 0x52, 0x45, 0x50, 0x4f, 0x53, 0x10, 0x1f,
-	0x12, 0x18, 0x0a, 0x14, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f,
-	0x54, 0x52, 0x41, 0x56, 0x49, 0x53, 0x43, 0x49, 0x10, 0x20, 0x12, 0x17, 0x0a, 0x13, 0x53, 0x4f,
-	0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x50, 0x4f, 0x53, 0x54, 0x4d, 0x41,
-	0x4e, 0x10, 0x21, 0x12, 0x17, 0x0a, 0x13, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59,
-	0x50, 0x45, 0x5f, 0x57, 0x45, 0x42, 0x48, 0x4f, 0x4f, 0x4b, 0x10, 0x22, 0x12, 0x1d, 0x0a, 0x19,
-	0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x45, 0x4c, 0x41, 0x53,
-	0x54, 0x49, 0x43, 0x53, 0x45, 0x41, 0x52, 0x43, 0x48, 0x10, 0x23, 0x12, 0x1b, 0x0a, 0x17, 0x53,
-	0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x48, 0x55, 0x47, 0x47, 0x49,
-	0x4e, 0x47, 0x46, 0x41, 0x43, 0x45, 0x10, 0x24, 0x12, 0x23, 0x0a, 0x1f, 0x53, 0x4f, 0x55, 0x52,
-	0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x49, 0x54, 0x48, 0x55, 0x42, 0x5f, 0x45,
-	0x58, 0x50, 0x45, 0x52, 0x49, 0x4d, 0x45, 0x4e, 0x54, 0x41, 0x4c, 0x10, 0x25, 0x12, 0x16, 0x0a,
-	0x12, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x45, 0x4e,
-	0x54, 0x52, 0x59, 0x10, 0x26, 0x12, 0x1f, 0x0a, 0x1b, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f,
-	0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x49, 0x54, 0x48, 0x55, 0x42, 0x5f, 0x52, 0x45, 0x41, 0x4c,
-	0x54, 0x49, 0x4d, 0x45, 0x10, 0x27, 0x12, 0x15, 0x0a, 0x11, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45,
-	0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x54, 0x44, 0x49, 0x4e, 0x10, 0x28, 0x12, 0x20, 0x0a,
-	0x1c, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x4c, 0x41,
-	0x43, 0x4b, 0x5f, 0x43, 0x4f, 0x4e, 0x54, 0x49, 0x4e, 0x55, 0x4f, 0x55, 0x53, 0x10, 0x29, 0x12,
-	0x1f, 0x0a, 0x1b, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4a,
-	0x53, 0x4f, 0x4e, 0x5f, 0x45, 0x4e, 0x55, 0x4d, 0x45, 0x52, 0x41, 0x54, 0x4f, 0x52, 0x10, 0x2a,
-	0x12, 0x13, 0x0a, 0x0f, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f,
-	0x57, 0x45, 0x42, 0x10, 0x2b, 0x2a, 0x47, 0x0a, 0x19, 0x42, 0x69, 0x74, 0x62, 0x75, 0x63, 0x6b,
-	0x65, 0x74, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79,
-	0x70, 0x65, 0x12, 0x0e, 0x0a, 0x0a, 0x41, 0x55, 0x54, 0x4f, 0x44, 0x45, 0x54, 0x45, 0x43, 0x54,
-	0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, 0x43, 0x4c, 0x4f, 0x55, 0x44, 0x10, 0x01, 0x12, 0x0f, 0x0a,
-	0x0b, 0x44, 0x41, 0x54, 0x41, 0x5f, 0x43, 0x45, 0x4e, 0x54, 0x45, 0x52, 0x10, 0x02, 0x2a, 0x87,
-	0x01, 0x0a, 0x14, 0x4a, 0x69, 0x72, 0x61, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x61, 0x74,
-	0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x12, 0x25, 0x0a, 0x21, 0x4a, 0x49, 0x52, 0x41, 0x5f,
-	0x49, 0x4e, 0x53, 0x54, 0x41, 0x4c, 0x4c, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x54, 0x59, 0x50,
-	0x45, 0x5f, 0x41, 0x55, 0x54, 0x4f, 0x44, 0x45, 0x54, 0x45, 0x43, 0x54, 0x10, 0x00, 0x12, 0x20,
-	0x0a, 0x1c, 0x4a, 0x49, 0x52, 0x41, 0x5f, 0x49, 0x4e, 0x53, 0x54, 0x41, 0x4c, 0x4c, 0x41, 0x54,
-	0x49, 0x4f, 0x4e, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x43, 0x4c, 0x4f, 0x55, 0x44, 0x10, 0x01,
-	0x12, 0x26, 0x0a, 0x22, 0x4a, 0x49, 0x52, 0x41, 0x5f, 0x49, 0x4e, 0x53, 0x54, 0x41, 0x4c, 0x4c,
-	0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x44, 0x41, 0x54, 0x41, 0x5f,
-	0x43, 0x45, 0x4e, 0x54, 0x45, 0x52, 0x10, 0x02, 0x42, 0x3b, 0x5a, 0x39, 0x67, 0x69, 0x74, 0x68,
-	0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73, 0x65,
-	0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68, 0x6f,
-	0x67, 0x2f, 0x76, 0x33, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x73, 0x6f, 0x75, 0x72,
-	0x63, 0x65, 0x73, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
+	0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x05, 0x70, 0x61, 0x74, 0x68, 0x73, 0x22, 0x9f, 0x01,
+	0x0a, 0x03, 0x57, 0x65, 0x62, 0x12, 0x12, 0x0a, 0x04, 0x75, 0x72, 0x6c, 0x73, 0x18, 0x01, 0x20,
+	0x03, 0x28, 0x09, 0x52, 0x04, 0x75, 0x72, 0x6c, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x63, 0x72, 0x61,
+	0x77, 0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, 0x63, 0x72, 0x61, 0x77, 0x6c, 0x12,
+	0x14, 0x0a, 0x05, 0x64, 0x65, 0x70, 0x74, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x05,
+	0x64, 0x65, 0x70, 0x74, 0x68, 0x12, 0x14, 0x0a, 0x05, 0x64, 0x65, 0x6c, 0x61, 0x79, 0x18, 0x04,
+	0x20, 0x01, 0x28, 0x03, 0x52, 0x05, 0x64, 0x65, 0x6c, 0x61, 0x79, 0x12, 0x1d, 0x0a, 0x0a, 0x75,
+	0x73, 0x65, 0x72, 0x5f, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52,
+	0x09, 0x75, 0x73, 0x65, 0x72, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x12, 0x23, 0x0a, 0x0d, 0x69, 0x67,
+	0x6e, 0x6f, 0x72, 0x65, 0x5f, 0x72, 0x6f, 0x62, 0x6f, 0x74, 0x73, 0x18, 0x06, 0x20, 0x01, 0x28,
+	0x08, 0x52, 0x0c, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x52, 0x6f, 0x62, 0x6f, 0x74, 0x73, 0x2a,
+	0xd9, 0x09, 0x0a, 0x0a, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x1d,
+	0x0a, 0x19, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x41, 0x5a,
+	0x55, 0x52, 0x45, 0x5f, 0x53, 0x54, 0x4f, 0x52, 0x41, 0x47, 0x45, 0x10, 0x00, 0x12, 0x19, 0x0a,
+	0x15, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x42, 0x49, 0x54,
+	0x42, 0x55, 0x43, 0x4b, 0x45, 0x54, 0x10, 0x01, 0x12, 0x18, 0x0a, 0x14, 0x53, 0x4f, 0x55, 0x52,
+	0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x43, 0x49, 0x52, 0x43, 0x4c, 0x45, 0x43, 0x49,
+	0x10, 0x02, 0x12, 0x1a, 0x0a, 0x16, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50,
+	0x45, 0x5f, 0x43, 0x4f, 0x4e, 0x46, 0x4c, 0x55, 0x45, 0x4e, 0x43, 0x45, 0x10, 0x03, 0x12, 0x16,
+	0x0a, 0x12, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x44, 0x4f,
+	0x43, 0x4b, 0x45, 0x52, 0x10, 0x04, 0x12, 0x13, 0x0a, 0x0f, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45,
+	0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x45, 0x43, 0x52, 0x10, 0x05, 0x12, 0x13, 0x0a, 0x0f, 0x53,
+	0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x43, 0x53, 0x10, 0x06,
+	0x12, 0x16, 0x0a, 0x12, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f,
+	0x47, 0x49, 0x54, 0x48, 0x55, 0x42, 0x10, 0x07, 0x12, 0x1a, 0x0a, 0x16, 0x53, 0x4f, 0x55, 0x52,
+	0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x50, 0x55, 0x42, 0x4c, 0x49, 0x43, 0x5f, 0x47,
+	0x49, 0x54, 0x10, 0x08, 0x12, 0x16, 0x0a, 0x12, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54,
+	0x59, 0x50, 0x45, 0x5f, 0x47, 0x49, 0x54, 0x4c, 0x41, 0x42, 0x10, 0x09, 0x12, 0x14, 0x0a, 0x10,
+	0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4a, 0x49, 0x52, 0x41,
+	0x10, 0x0a, 0x12, 0x24, 0x0a, 0x20, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50,
+	0x45, 0x5f, 0x4e, 0x50, 0x4d, 0x5f, 0x55, 0x4e, 0x41, 0x55, 0x54, 0x48, 0x44, 0x5f, 0x50, 0x41,
+	0x43, 0x4b, 0x41, 0x47, 0x45, 0x53, 0x10, 0x0b, 0x12, 0x25, 0x0a, 0x21, 0x53, 0x4f, 0x55, 0x52,
+	0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x50, 0x59, 0x50, 0x49, 0x5f, 0x55, 0x4e, 0x41,
+	0x55, 0x54, 0x48, 0x44, 0x5f, 0x50, 0x41, 0x43, 0x4b, 0x41, 0x47, 0x45, 0x53, 0x10, 0x0c, 0x12,
+	0x12, 0x0a, 0x0e, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53,
+	0x33, 0x10, 0x0d, 0x12, 0x15, 0x0a, 0x11, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59,
+	0x50, 0x45, 0x5f, 0x53, 0x4c, 0x41, 0x43, 0x4b, 0x10, 0x0e, 0x12, 0x1a, 0x0a, 0x16, 0x53, 0x4f,
+	0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x46, 0x49, 0x4c, 0x45, 0x53, 0x59,
+	0x53, 0x54, 0x45, 0x4d, 0x10, 0x0f, 0x12, 0x13, 0x0a, 0x0f, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45,
+	0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x49, 0x54, 0x10, 0x10, 0x12, 0x14, 0x0a, 0x10, 0x53,
+	0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x54, 0x45, 0x53, 0x54, 0x10,
+	0x11, 0x12, 0x1b, 0x0a, 0x17, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45,
+	0x5f, 0x53, 0x33, 0x5f, 0x55, 0x4e, 0x41, 0x55, 0x54, 0x48, 0x45, 0x44, 0x10, 0x12, 0x12, 0x2a,
+	0x0a, 0x26, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x49,
+	0x54, 0x48, 0x55, 0x42, 0x5f, 0x55, 0x4e, 0x41, 0x55, 0x54, 0x48, 0x45, 0x4e, 0x54, 0x49, 0x43,
+	0x41, 0x54, 0x45, 0x44, 0x5f, 0x4f, 0x52, 0x47, 0x10, 0x13, 0x12, 0x19, 0x0a, 0x15, 0x53, 0x4f,
+	0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x42, 0x55, 0x49, 0x4c, 0x44, 0x4b,
+	0x49, 0x54, 0x45, 0x10, 0x14, 0x12, 0x16, 0x0a, 0x12, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f,
+	0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x45, 0x52, 0x52, 0x49, 0x54, 0x10, 0x15, 0x12, 0x17, 0x0a,
+	0x13, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4a, 0x45, 0x4e,
+	0x4b, 0x49, 0x4e, 0x53, 0x10, 0x16, 0x12, 0x15, 0x0a, 0x11, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45,
+	0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x54, 0x45, 0x41, 0x4d, 0x53, 0x10, 0x17, 0x12, 0x21, 0x0a,
+	0x1d, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4a, 0x46, 0x52,
+	0x4f, 0x47, 0x5f, 0x41, 0x52, 0x54, 0x49, 0x46, 0x41, 0x43, 0x54, 0x4f, 0x52, 0x59, 0x10, 0x18,
+	0x12, 0x16, 0x0a, 0x12, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f,
+	0x53, 0x59, 0x53, 0x4c, 0x4f, 0x47, 0x10, 0x19, 0x12, 0x27, 0x0a, 0x23, 0x53, 0x4f, 0x55, 0x52,
+	0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x50, 0x55, 0x42, 0x4c, 0x49, 0x43, 0x5f, 0x45,
+	0x56, 0x45, 0x4e, 0x54, 0x5f, 0x4d, 0x4f, 0x4e, 0x49, 0x54, 0x4f, 0x52, 0x49, 0x4e, 0x47, 0x10,
+	0x1a, 0x12, 0x1e, 0x0a, 0x1a, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45,
+	0x5f, 0x53, 0x4c, 0x41, 0x43, 0x4b, 0x5f, 0x52, 0x45, 0x41, 0x4c, 0x54, 0x49, 0x4d, 0x45, 0x10,
+	0x1b, 0x12, 0x1c, 0x0a, 0x18, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45,
+	0x5f, 0x47, 0x4f, 0x4f, 0x47, 0x4c, 0x45, 0x5f, 0x44, 0x52, 0x49, 0x56, 0x45, 0x10, 0x1c, 0x12,
+	0x1a, 0x0a, 0x16, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53,
+	0x48, 0x41, 0x52, 0x45, 0x50, 0x4f, 0x49, 0x4e, 0x54, 0x10, 0x1d, 0x12, 0x1c, 0x0a, 0x18, 0x53,
+	0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x43, 0x53, 0x5f, 0x55,
+	0x4e, 0x41, 0x55, 0x54, 0x48, 0x45, 0x44, 0x10, 0x1e, 0x12, 0x1b, 0x0a, 0x17, 0x53, 0x4f, 0x55,
+	0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x41, 0x5a, 0x55, 0x52, 0x45, 0x5f, 0x52,
+	0x45, 0x50, 0x4f, 0x53, 0x10, 0x1f, 0x12, 0x18, 0x0a, 0x14, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45,
+	0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x54, 0x52, 0x41, 0x56, 0x49, 0x53, 0x43, 0x49, 0x10, 0x20,
+	0x12, 0x17, 0x0a, 0x13, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f,
+	0x50, 0x4f, 0x53, 0x54, 0x4d, 0x41, 0x4e, 0x10, 0x21, 0x12, 0x17, 0x0a, 0x13, 0x53, 0x4f, 0x55,
+	0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x57, 0x45, 0x42, 0x48, 0x4f, 0x4f, 0x4b,
+	0x10, 0x22, 0x12, 0x1d, 0x0a, 0x19, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50,
+	0x45, 0x5f, 0x45, 0x4c, 0x41, 0x53, 0x54, 0x49, 0x43, 0x53, 0x45, 0x41, 0x52, 0x43, 0x48, 0x10,
+	0x23, 0x12, 0x1b, 0x0a, 0x17, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45,
+	0x5f, 0x48, 0x55, 0x47, 0x47, 0x49, 0x4e, 0x47, 0x46, 0x41, 0x43, 0x45, 0x10, 0x24, 0x12, 0x23,
+	0x0a, 0x1f, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x49,
+	0x54, 0x48, 0x55, 0x42, 0x5f, 0x45, 0x58, 0x50, 0x45, 0x52, 0x49, 0x4d, 0x45, 0x4e, 0x54, 0x41,
+	0x4c, 0x10, 0x25, 0x12, 0x16, 0x0a, 0x12, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59,
+	0x50, 0x45, 0x5f, 0x53, 0x45, 0x4e, 0x54, 0x52, 0x59, 0x10, 0x26, 0x12, 0x1f, 0x0a, 0x1b, 0x53,
+	0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x49, 0x54, 0x48, 0x55,
+	0x42, 0x5f, 0x52, 0x45, 0x41, 0x4c, 0x54, 0x49, 0x4d, 0x45, 0x10, 0x27, 0x12, 0x15, 0x0a, 0x11,
+	0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x54, 0x44, 0x49,
+	0x4e, 0x10, 0x28, 0x12, 0x20, 0x0a, 0x1c, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59,
+	0x50, 0x45, 0x5f, 0x53, 0x4c, 0x41, 0x43, 0x4b, 0x5f, 0x43, 0x4f, 0x4e, 0x54, 0x49, 0x4e, 0x55,
+	0x4f, 0x55, 0x53, 0x10, 0x29, 0x12, 0x1f, 0x0a, 0x1b, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f,
+	0x54, 0x59, 0x50, 0x45, 0x5f, 0x4a, 0x53, 0x4f, 0x4e, 0x5f, 0x45, 0x4e, 0x55, 0x4d, 0x45, 0x52,
+	0x41, 0x54, 0x4f, 0x52, 0x10, 0x2a, 0x12, 0x13, 0x0a, 0x0f, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45,
+	0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x57, 0x45, 0x42, 0x10, 0x2b, 0x2a, 0x47, 0x0a, 0x19, 0x42,
+	0x69, 0x74, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x61,
+	0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0e, 0x0a, 0x0a, 0x41, 0x55, 0x54, 0x4f,
+	0x44, 0x45, 0x54, 0x45, 0x43, 0x54, 0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, 0x43, 0x4c, 0x4f, 0x55,
+	0x44, 0x10, 0x01, 0x12, 0x0f, 0x0a, 0x0b, 0x44, 0x41, 0x54, 0x41, 0x5f, 0x43, 0x45, 0x4e, 0x54,
+	0x45, 0x52, 0x10, 0x02, 0x2a, 0x87, 0x01, 0x0a, 0x14, 0x4a, 0x69, 0x72, 0x61, 0x49, 0x6e, 0x73,
+	0x74, 0x61, 0x6c, 0x6c, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x12, 0x25, 0x0a,
+	0x21, 0x4a, 0x49, 0x52, 0x41, 0x5f, 0x49, 0x4e, 0x53, 0x54, 0x41, 0x4c, 0x4c, 0x41, 0x54, 0x49,
+	0x4f, 0x4e, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x41, 0x55, 0x54, 0x4f, 0x44, 0x45, 0x54, 0x45,
+	0x43, 0x54, 0x10, 0x00, 0x12, 0x20, 0x0a, 0x1c, 0x4a, 0x49, 0x52, 0x41, 0x5f, 0x49, 0x4e, 0x53,
+	0x54, 0x41, 0x4c, 0x4c, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x43,
+	0x4c, 0x4f, 0x55, 0x44, 0x10, 0x01, 0x12, 0x26, 0x0a, 0x22, 0x4a, 0x49, 0x52, 0x41, 0x5f, 0x49,
+	0x4e, 0x53, 0x54, 0x41, 0x4c, 0x4c, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x54, 0x59, 0x50, 0x45,
+	0x5f, 0x44, 0x41, 0x54, 0x41, 0x5f, 0x43, 0x45, 0x4e, 0x54, 0x45, 0x52, 0x10, 0x02, 0x42, 0x3b,
+	0x5a, 0x39, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x72, 0x75,
+	0x66, 0x66, 0x6c, 0x65, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x2f, 0x74, 0x72, 0x75,
+	0x66, 0x66, 0x6c, 0x65, 0x68, 0x6f, 0x67, 0x2f, 0x76, 0x33, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70,
+	0x62, 0x2f, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f,
+	0x74, 0x6f, 0x33,
 }
 
 var (
diff --git a/pkg/pb/sourcespb/sources.pb.validate.go b/pkg/pb/sourcespb/sources.pb.validate.go
index 823472b035bb..fab83443aa12 100644
--- a/pkg/pb/sourcespb/sources.pb.validate.go
+++ b/pkg/pb/sourcespb/sources.pb.validate.go
@@ -7268,6 +7268,8 @@ func (m *Web) validate(all bool) error {
 
 	// no validation rules for UserAgent
 
+	// no validation rules for IgnoreRobots
+
 	if len(errors) > 0 {
 		return WebMultiError(errors)
 	}
diff --git a/pkg/sources/sources.go b/pkg/sources/sources.go
index a5cb54129bd8..f55b62b68f62 100644
--- a/pkg/sources/sources.go
+++ b/pkg/sources/sources.go
@@ -495,8 +495,9 @@ type WebConfig struct {
 	Depth int
 
 	// Delay is the delay (in seconds) between requests to the same domain.
-	Delay     int
-	UserAgent string
+	Delay        int
+	UserAgent    string
+	IgnoreRobots bool
 }
 
 // Progress is used to update job completion progress across sources.
diff --git a/pkg/sources/web/web.go b/pkg/sources/web/web.go
index d29daa285809..2ef1a294577a 100644
--- a/pkg/sources/web/web.go
+++ b/pkg/sources/web/web.go
@@ -67,6 +67,10 @@ func (s *Source) Init(ctx context.Context, name string, jobId sources.JobID, sou
 		s.conn.UserAgent = "trufflehog-web (+https://github.com/trufflesecurity/trufflehog)"
 	}
 
+	if s.conn.GetIgnoreRobots() {
+		ctx.Logger().Info("Warning: Robots.txt is ignored. Only use this if you have permission to crawl the target site.")
+	}
+
 	// validations
 	if len(s.conn.GetUrls()) == 0 {
 		return errors.New("no URL provided")
@@ -112,6 +116,13 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ .
 }
 
 func (s *Source) crawlURL(ctx context.Context, seedURL string, chunksChan chan *sources.Chunk) error {
+	// Add static crawl configuration to the context so that all subsequent logs include these fields.
+	ctx = context.WithValues(ctx,
+		"url", seedURL,
+		"user_agent", s.conn.GetUserAgent(),
+		"ignore_robots", s.conn.GetIgnoreRobots(),
+	)
+
 	url, err := url.Parse(seedURL)
 	if err != nil {
 		return fmt.Errorf("invalid URL %q: %w", seedURL, err)
@@ -123,9 +134,9 @@ func (s *Source) crawlURL(ctx context.Context, seedURL string, chunksChan chan *
 		colly.Async(true),
 	)
 
-	// Respect robots.txt
-	// TODO: maybe allow users to set this as well with warning
-	collector.IgnoreRobotsTxt = false
+	// By default, the crawler respects robots.txt rules. Setting IgnoreRobotsTxt to true overrides this behavior.
+	// Users can enable this only when they have explicit permission to crawl the site.
+	collector.IgnoreRobotsTxt = s.conn.GetIgnoreRobots()
 
 	collector.Limit(&colly.LimitRule{
 		DomainGlob:  "*",
@@ -135,7 +146,7 @@ func (s *Source) crawlURL(ctx context.Context, seedURL string, chunksChan chan *
 
 	// Set up callbacks
 	collector.OnResponse(func(r *colly.Response) {
-		ctx.Logger().Info("OnResponse fired", "url", r.Request.URL)
+		ctx.Logger().Info("Response recieved")
 		if err := s.processChunk(ctx, r, chunksChan); err != nil {
 			ctx.Logger().Error(err, "error processing page")
 		}
@@ -147,7 +158,7 @@ func (s *Source) crawlURL(ctx context.Context, seedURL string, chunksChan chan *
 	// Create a channel to signal when the crawl is done.
 	done := make(chan struct{})
 	go func() {
-		ctx.Logger().Info("starting crawl")
+		ctx.Logger().Info("Starting crawl")
 		if err := collector.Visit(seedURL); err != nil {
 			ctx.Logger().Error(err, "Visit failed")
 		}
@@ -158,10 +169,10 @@ func (s *Source) crawlURL(ctx context.Context, seedURL string, chunksChan chan *
 	// Wait for either crawl to finish or context cancellation.
 	select {
 	case <-done:
-		ctx.Logger().Info("crawl finished normally")
+		ctx.Logger().Info("Crawl finished normally")
 		return nil
 	case <-ctx.Done():
-		ctx.Logger().Info("context cancelled or timeout reached")
+		ctx.Logger().Info("Context cancelled or timeout reached")
 		<-done // Wait for goroutine to finish cleanup
 		return ctx.Err()
 	}
diff --git a/proto/sources.proto b/proto/sources.proto
index 1712150eba58..18f28aacf5c6 100644
--- a/proto/sources.proto
+++ b/proto/sources.proto
@@ -562,4 +562,5 @@ message Web {
   int64 depth = 3;
   int64 delay = 4;
   string user_agent = 5;
+  bool ignore_robots = 6;
 }

From 17a6e2c4b478a8a692e7db01dde3806ee61e0927 Mon Sep 17 00:00:00 2001
From: Kashif Khan <kashif.khan@trufflesec.com>
Date: Mon, 30 Mar 2026 15:18:47 +0500
Subject: [PATCH 07/20] added metric

---
 main.go                    |  5 -----
 pkg/sources/web/metrics.go | 17 +++++++++++++++++
 pkg/sources/web/web.go     |  8 +++++++-
 3 files changed, 24 insertions(+), 6 deletions(-)
 create mode 100644 pkg/sources/web/metrics.go

diff --git a/main.go b/main.go
index 2cd682b36fde..6a342247193e 100644
--- a/main.go
+++ b/main.go
@@ -1169,11 +1169,6 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics,
 			return scanMetrics, fmt.Errorf("invalid config: you must specify at least one url")
 		}
 
-		if *webUserAgent == "" {
-			ctx.Logger().Info("No user agent set; using default", "user-agent", "trufflehog-web (+https://github.com/trufflesecurity/trufflehog)")
-			*webUserAgent = "trufflehog-web (+https://github.com/trufflesecurity/trufflehog)"
-		}
-
 		cfg := sources.WebConfig{
 			URLs:         *webUrls,
 			Crawl:        *webCrawl,
diff --git a/pkg/sources/web/metrics.go b/pkg/sources/web/metrics.go
new file mode 100644
index 000000000000..9f8d85e2a00c
--- /dev/null
+++ b/pkg/sources/web/metrics.go
@@ -0,0 +1,17 @@
+package web
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+)
+
+var (
+	webUrlsScanned = promauto.NewGaugeVec(prometheus.GaugeOpts{
+		Namespace: common.MetricsNamespace,
+		Subsystem: common.MetricsSubsystem,
+		Name:      "web_urls_scanned",
+		Help:      "Total number of URLs scanned.",
+	},
+		[]string{"source_name", "job_id"})
+)
diff --git a/pkg/sources/web/web.go b/pkg/sources/web/web.go
index 2ef1a294577a..35b1b3d5f470 100644
--- a/pkg/sources/web/web.go
+++ b/pkg/sources/web/web.go
@@ -64,6 +64,7 @@ func (s *Source) Init(ctx context.Context, name string, jobId sources.JobID, sou
 
 	// Use the user-provided User-Agent if set; otherwise fall back to a default that identifies TruffleHog.
 	if s.conn.GetUserAgent() == "" {
+		ctx.Logger().Info("No user agent set; using default", "user-agent", "trufflehog-web (+https://github.com/trufflesecurity/trufflehog)")
 		s.conn.UserAgent = "trufflehog-web (+https://github.com/trufflesecurity/trufflehog)"
 	}
 
@@ -87,7 +88,9 @@ func (s *Source) Init(ctx context.Context, name string, jobId sources.JobID, sou
 		}
 	}
 
-	// TODO: reset metrics if needed
+	// metrics
+	jobIDStr := fmt.Sprint(s.jobId)
+	webUrlsScanned.WithLabelValues(s.name, jobIDStr).Set(0)
 
 	return nil
 }
@@ -95,6 +98,7 @@ func (s *Source) Init(ctx context.Context, name string, jobId sources.JobID, sou
 // Chunks emits data over a channel that is decoded and scanned for secrets.
 func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ ...sources.ChunkingTarget) error {
 	var wg sync.WaitGroup
+	jobIDStr := fmt.Sprint(s.jobId)
 
 	// Create a background context for crawling (independent of incoming ctx)
 	crawlCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
@@ -107,6 +111,8 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ .
 			defer wg.Done()
 			s.crawlURL(crawlCtx, url, chunksChan)
 		}(url)
+
+		webUrlsScanned.WithLabelValues(s.name, jobIDStr).Inc()
 	}
 
 	// Block until all crawls complete

From 605c14ead0ffab650a23d9e705242f345fbebd0a Mon Sep 17 00:00:00 2001
From: Kashif Khan <kashif.khan@trufflesec.com>
Date: Mon, 30 Mar 2026 16:18:22 +0500
Subject: [PATCH 08/20] detailed test cases

---
 main.go                                 |  16 +-
 pkg/pb/sourcespb/sources.pb.go          | 217 ++++++------
 pkg/pb/sourcespb/sources.pb.validate.go |   2 +
 pkg/sources/sources.go                  |  28 +-
 pkg/sources/web/web.go                  |  81 +++--
 pkg/sources/web/web_test.go             | 431 +++++++++++++++++++++---
 proto/sources.proto                     |   5 +-
 7 files changed, 601 insertions(+), 179 deletions(-)

diff --git a/main.go b/main.go
index 6a342247193e..9fad356824c0 100644
--- a/main.go
+++ b/main.go
@@ -276,13 +276,14 @@ var (
 	jsonEnumeratorScan  = cli.Command("json-enumerator", "Find credentials from a JSON enumerator input.")
 	jsonEnumeratorPaths = jsonEnumeratorScan.Arg("path", "Path to JSON enumerator file to scan.").Strings()
 
-	webScan         = cli.Command("web", "Scan websites for leaked credentials")
-	webUrls         = webScan.Flag("url", "One or more URLs to scan (required). You can repeat this flag. Supports http:// and https://.").Required().Strings()
-	webCrawl        = webScan.Flag("crawl", "Enable crawling: follow links discovered on the initial page(s).").Default("false").Bool()
-	webDepth        = webScan.Flag("depth", "Maximum link depth to follow when crawling. 0 = only the seed URL(s); 1 = seed + direct links; etc.").Default("1").Int()
-	webDelay        = webScan.Flag("delay", "Delay (in seconds) between requests to the same domain. Helps respect server load.").Default("1").Int()
-	webUserAgent    = webScan.Flag("user-agent", "User-Agent header sent with each HTTP request. If not set, a descriptive default is used.").String()
-	webIgnoreRobots = webScan.Flag("ignore-robots", "Ignore robots.txt rules. Use only if you have permission to crawl the site, otherwise this may violate the site's policies.").Default("false").Bool()
+	webScan         = cli.Command("web", "Scan websites for leaked credentials.")
+	webUrls         = webScan.Flag("url", "URL to scan. Repeat the flag for multiple targets, e.g. --url https://a.com --url https://b.com. Supports http:// and https://.").Required().Strings()
+	webCrawl        = webScan.Flag("crawl", "Follow links found on each page. Without this flag only the seed URL(s) are scanned.").Default("false").Bool()
+	webDepth        = webScan.Flag("depth", "Maximum link depth to follow when --crawl is enabled. 1 = seed + direct links; 2 = one level deeper; 0 = unlimited.").Default("1").Int()
+	webDelay        = webScan.Flag("delay", "Seconds to wait between requests to the same domain. Increase this to reduce load on the target server.").Default("1").Int()
+	webTimeout      = webScan.Flag("timeout", "Seconds to spend crawling each URL before aborting. Applied per URL when multiple --url flags are given.").Default("30").Int()
+	webUserAgent    = webScan.Flag("user-agent", "User-Agent header to send with each request. Defaults to a TruffleHog identifier if not set.").String()
+	webIgnoreRobots = webScan.Flag("ignore-robots", "Ignore robots.txt restrictions. Only use this if you have explicit permission to crawl the target site.").Default("false").Bool()
 
 	analyzeCmd = analyzer.Command(cli)
 	usingTUI   = false
@@ -1174,6 +1175,7 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics,
 			Crawl:        *webCrawl,
 			Depth:        *webDepth,
 			Delay:        *webDelay,
+			Timeout:      *webTimeout,
 			UserAgent:    *webUserAgent,
 			IgnoreRobots: *webIgnoreRobots,
 		}
diff --git a/pkg/pb/sourcespb/sources.pb.go b/pkg/pb/sourcespb/sources.pb.go
index 08fd94b6a066..55eefc9de7f7 100644
--- a/pkg/pb/sourcespb/sources.pb.go
+++ b/pkg/pb/sourcespb/sources.pb.go
@@ -5112,8 +5112,9 @@ type Web struct {
 	Crawl        bool     `protobuf:"varint,2,opt,name=crawl,proto3" json:"crawl,omitempty"`
 	Depth        int64    `protobuf:"varint,3,opt,name=depth,proto3" json:"depth,omitempty"`
 	Delay        int64    `protobuf:"varint,4,opt,name=delay,proto3" json:"delay,omitempty"`
-	UserAgent    string   `protobuf:"bytes,5,opt,name=user_agent,json=userAgent,proto3" json:"user_agent,omitempty"`
-	IgnoreRobots bool     `protobuf:"varint,6,opt,name=ignore_robots,json=ignoreRobots,proto3" json:"ignore_robots,omitempty"`
+	Timeout      int64    `protobuf:"varint,5,opt,name=timeout,proto3" json:"timeout,omitempty"`
+	UserAgent    string   `protobuf:"bytes,6,opt,name=user_agent,json=userAgent,proto3" json:"user_agent,omitempty"`
+	IgnoreRobots bool     `protobuf:"varint,7,opt,name=ignore_robots,json=ignoreRobots,proto3" json:"ignore_robots,omitempty"`
 }
 
 func (x *Web) Reset() {
@@ -5176,6 +5177,13 @@ func (x *Web) GetDelay() int64 {
 	return 0
 }
 
+func (x *Web) GetTimeout() int64 {
+	if x != nil {
+		return x.Timeout
+	}
+	return 0
+}
+
 func (x *Web) GetUserAgent() string {
 	if x != nil {
 		return x.UserAgent
@@ -5982,113 +5990,114 @@ var file_sources_proto_rawDesc = []byte{
 	0x74, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x70, 0x72, 0x6f, 0x6a,
 	0x65, 0x63, 0x74, 0x49, 0x64, 0x22, 0x26, 0x0a, 0x0e, 0x4a, 0x53, 0x4f, 0x4e, 0x45, 0x6e, 0x75,
 	0x6d, 0x65, 0x72, 0x61, 0x74, 0x6f, 0x72, 0x12, 0x14, 0x0a, 0x05, 0x70, 0x61, 0x74, 0x68, 0x73,
-	0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x05, 0x70, 0x61, 0x74, 0x68, 0x73, 0x22, 0x9f, 0x01,
+	0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x05, 0x70, 0x61, 0x74, 0x68, 0x73, 0x22, 0xb9, 0x01,
 	0x0a, 0x03, 0x57, 0x65, 0x62, 0x12, 0x12, 0x0a, 0x04, 0x75, 0x72, 0x6c, 0x73, 0x18, 0x01, 0x20,
 	0x03, 0x28, 0x09, 0x52, 0x04, 0x75, 0x72, 0x6c, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x63, 0x72, 0x61,
 	0x77, 0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, 0x63, 0x72, 0x61, 0x77, 0x6c, 0x12,
 	0x14, 0x0a, 0x05, 0x64, 0x65, 0x70, 0x74, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x05,
 	0x64, 0x65, 0x70, 0x74, 0x68, 0x12, 0x14, 0x0a, 0x05, 0x64, 0x65, 0x6c, 0x61, 0x79, 0x18, 0x04,
-	0x20, 0x01, 0x28, 0x03, 0x52, 0x05, 0x64, 0x65, 0x6c, 0x61, 0x79, 0x12, 0x1d, 0x0a, 0x0a, 0x75,
-	0x73, 0x65, 0x72, 0x5f, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52,
-	0x09, 0x75, 0x73, 0x65, 0x72, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x12, 0x23, 0x0a, 0x0d, 0x69, 0x67,
-	0x6e, 0x6f, 0x72, 0x65, 0x5f, 0x72, 0x6f, 0x62, 0x6f, 0x74, 0x73, 0x18, 0x06, 0x20, 0x01, 0x28,
-	0x08, 0x52, 0x0c, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x52, 0x6f, 0x62, 0x6f, 0x74, 0x73, 0x2a,
-	0xd9, 0x09, 0x0a, 0x0a, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x1d,
-	0x0a, 0x19, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x41, 0x5a,
-	0x55, 0x52, 0x45, 0x5f, 0x53, 0x54, 0x4f, 0x52, 0x41, 0x47, 0x45, 0x10, 0x00, 0x12, 0x19, 0x0a,
-	0x15, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x42, 0x49, 0x54,
-	0x42, 0x55, 0x43, 0x4b, 0x45, 0x54, 0x10, 0x01, 0x12, 0x18, 0x0a, 0x14, 0x53, 0x4f, 0x55, 0x52,
-	0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x43, 0x49, 0x52, 0x43, 0x4c, 0x45, 0x43, 0x49,
-	0x10, 0x02, 0x12, 0x1a, 0x0a, 0x16, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50,
-	0x45, 0x5f, 0x43, 0x4f, 0x4e, 0x46, 0x4c, 0x55, 0x45, 0x4e, 0x43, 0x45, 0x10, 0x03, 0x12, 0x16,
-	0x0a, 0x12, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x44, 0x4f,
-	0x43, 0x4b, 0x45, 0x52, 0x10, 0x04, 0x12, 0x13, 0x0a, 0x0f, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45,
-	0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x45, 0x43, 0x52, 0x10, 0x05, 0x12, 0x13, 0x0a, 0x0f, 0x53,
-	0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x43, 0x53, 0x10, 0x06,
-	0x12, 0x16, 0x0a, 0x12, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f,
-	0x47, 0x49, 0x54, 0x48, 0x55, 0x42, 0x10, 0x07, 0x12, 0x1a, 0x0a, 0x16, 0x53, 0x4f, 0x55, 0x52,
-	0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x50, 0x55, 0x42, 0x4c, 0x49, 0x43, 0x5f, 0x47,
-	0x49, 0x54, 0x10, 0x08, 0x12, 0x16, 0x0a, 0x12, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54,
-	0x59, 0x50, 0x45, 0x5f, 0x47, 0x49, 0x54, 0x4c, 0x41, 0x42, 0x10, 0x09, 0x12, 0x14, 0x0a, 0x10,
-	0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4a, 0x49, 0x52, 0x41,
-	0x10, 0x0a, 0x12, 0x24, 0x0a, 0x20, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50,
-	0x45, 0x5f, 0x4e, 0x50, 0x4d, 0x5f, 0x55, 0x4e, 0x41, 0x55, 0x54, 0x48, 0x44, 0x5f, 0x50, 0x41,
-	0x43, 0x4b, 0x41, 0x47, 0x45, 0x53, 0x10, 0x0b, 0x12, 0x25, 0x0a, 0x21, 0x53, 0x4f, 0x55, 0x52,
-	0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x50, 0x59, 0x50, 0x49, 0x5f, 0x55, 0x4e, 0x41,
-	0x55, 0x54, 0x48, 0x44, 0x5f, 0x50, 0x41, 0x43, 0x4b, 0x41, 0x47, 0x45, 0x53, 0x10, 0x0c, 0x12,
-	0x12, 0x0a, 0x0e, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53,
-	0x33, 0x10, 0x0d, 0x12, 0x15, 0x0a, 0x11, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59,
-	0x50, 0x45, 0x5f, 0x53, 0x4c, 0x41, 0x43, 0x4b, 0x10, 0x0e, 0x12, 0x1a, 0x0a, 0x16, 0x53, 0x4f,
-	0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x46, 0x49, 0x4c, 0x45, 0x53, 0x59,
-	0x53, 0x54, 0x45, 0x4d, 0x10, 0x0f, 0x12, 0x13, 0x0a, 0x0f, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45,
-	0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x49, 0x54, 0x10, 0x10, 0x12, 0x14, 0x0a, 0x10, 0x53,
-	0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x54, 0x45, 0x53, 0x54, 0x10,
-	0x11, 0x12, 0x1b, 0x0a, 0x17, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45,
-	0x5f, 0x53, 0x33, 0x5f, 0x55, 0x4e, 0x41, 0x55, 0x54, 0x48, 0x45, 0x44, 0x10, 0x12, 0x12, 0x2a,
-	0x0a, 0x26, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x49,
-	0x54, 0x48, 0x55, 0x42, 0x5f, 0x55, 0x4e, 0x41, 0x55, 0x54, 0x48, 0x45, 0x4e, 0x54, 0x49, 0x43,
-	0x41, 0x54, 0x45, 0x44, 0x5f, 0x4f, 0x52, 0x47, 0x10, 0x13, 0x12, 0x19, 0x0a, 0x15, 0x53, 0x4f,
-	0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x42, 0x55, 0x49, 0x4c, 0x44, 0x4b,
-	0x49, 0x54, 0x45, 0x10, 0x14, 0x12, 0x16, 0x0a, 0x12, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f,
-	0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x45, 0x52, 0x52, 0x49, 0x54, 0x10, 0x15, 0x12, 0x17, 0x0a,
-	0x13, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4a, 0x45, 0x4e,
-	0x4b, 0x49, 0x4e, 0x53, 0x10, 0x16, 0x12, 0x15, 0x0a, 0x11, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45,
-	0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x54, 0x45, 0x41, 0x4d, 0x53, 0x10, 0x17, 0x12, 0x21, 0x0a,
-	0x1d, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4a, 0x46, 0x52,
-	0x4f, 0x47, 0x5f, 0x41, 0x52, 0x54, 0x49, 0x46, 0x41, 0x43, 0x54, 0x4f, 0x52, 0x59, 0x10, 0x18,
-	0x12, 0x16, 0x0a, 0x12, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f,
-	0x53, 0x59, 0x53, 0x4c, 0x4f, 0x47, 0x10, 0x19, 0x12, 0x27, 0x0a, 0x23, 0x53, 0x4f, 0x55, 0x52,
-	0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x50, 0x55, 0x42, 0x4c, 0x49, 0x43, 0x5f, 0x45,
-	0x56, 0x45, 0x4e, 0x54, 0x5f, 0x4d, 0x4f, 0x4e, 0x49, 0x54, 0x4f, 0x52, 0x49, 0x4e, 0x47, 0x10,
-	0x1a, 0x12, 0x1e, 0x0a, 0x1a, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45,
-	0x5f, 0x53, 0x4c, 0x41, 0x43, 0x4b, 0x5f, 0x52, 0x45, 0x41, 0x4c, 0x54, 0x49, 0x4d, 0x45, 0x10,
-	0x1b, 0x12, 0x1c, 0x0a, 0x18, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45,
-	0x5f, 0x47, 0x4f, 0x4f, 0x47, 0x4c, 0x45, 0x5f, 0x44, 0x52, 0x49, 0x56, 0x45, 0x10, 0x1c, 0x12,
-	0x1a, 0x0a, 0x16, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53,
-	0x48, 0x41, 0x52, 0x45, 0x50, 0x4f, 0x49, 0x4e, 0x54, 0x10, 0x1d, 0x12, 0x1c, 0x0a, 0x18, 0x53,
-	0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x43, 0x53, 0x5f, 0x55,
-	0x4e, 0x41, 0x55, 0x54, 0x48, 0x45, 0x44, 0x10, 0x1e, 0x12, 0x1b, 0x0a, 0x17, 0x53, 0x4f, 0x55,
-	0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x41, 0x5a, 0x55, 0x52, 0x45, 0x5f, 0x52,
-	0x45, 0x50, 0x4f, 0x53, 0x10, 0x1f, 0x12, 0x18, 0x0a, 0x14, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45,
-	0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x54, 0x52, 0x41, 0x56, 0x49, 0x53, 0x43, 0x49, 0x10, 0x20,
-	0x12, 0x17, 0x0a, 0x13, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f,
-	0x50, 0x4f, 0x53, 0x54, 0x4d, 0x41, 0x4e, 0x10, 0x21, 0x12, 0x17, 0x0a, 0x13, 0x53, 0x4f, 0x55,
-	0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x57, 0x45, 0x42, 0x48, 0x4f, 0x4f, 0x4b,
-	0x10, 0x22, 0x12, 0x1d, 0x0a, 0x19, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50,
-	0x45, 0x5f, 0x45, 0x4c, 0x41, 0x53, 0x54, 0x49, 0x43, 0x53, 0x45, 0x41, 0x52, 0x43, 0x48, 0x10,
-	0x23, 0x12, 0x1b, 0x0a, 0x17, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45,
-	0x5f, 0x48, 0x55, 0x47, 0x47, 0x49, 0x4e, 0x47, 0x46, 0x41, 0x43, 0x45, 0x10, 0x24, 0x12, 0x23,
-	0x0a, 0x1f, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x49,
-	0x54, 0x48, 0x55, 0x42, 0x5f, 0x45, 0x58, 0x50, 0x45, 0x52, 0x49, 0x4d, 0x45, 0x4e, 0x54, 0x41,
-	0x4c, 0x10, 0x25, 0x12, 0x16, 0x0a, 0x12, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59,
-	0x50, 0x45, 0x5f, 0x53, 0x45, 0x4e, 0x54, 0x52, 0x59, 0x10, 0x26, 0x12, 0x1f, 0x0a, 0x1b, 0x53,
-	0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x49, 0x54, 0x48, 0x55,
-	0x42, 0x5f, 0x52, 0x45, 0x41, 0x4c, 0x54, 0x49, 0x4d, 0x45, 0x10, 0x27, 0x12, 0x15, 0x0a, 0x11,
-	0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x54, 0x44, 0x49,
-	0x4e, 0x10, 0x28, 0x12, 0x20, 0x0a, 0x1c, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59,
-	0x50, 0x45, 0x5f, 0x53, 0x4c, 0x41, 0x43, 0x4b, 0x5f, 0x43, 0x4f, 0x4e, 0x54, 0x49, 0x4e, 0x55,
-	0x4f, 0x55, 0x53, 0x10, 0x29, 0x12, 0x1f, 0x0a, 0x1b, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f,
-	0x54, 0x59, 0x50, 0x45, 0x5f, 0x4a, 0x53, 0x4f, 0x4e, 0x5f, 0x45, 0x4e, 0x55, 0x4d, 0x45, 0x52,
-	0x41, 0x54, 0x4f, 0x52, 0x10, 0x2a, 0x12, 0x13, 0x0a, 0x0f, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45,
-	0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x57, 0x45, 0x42, 0x10, 0x2b, 0x2a, 0x47, 0x0a, 0x19, 0x42,
-	0x69, 0x74, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x61,
-	0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0e, 0x0a, 0x0a, 0x41, 0x55, 0x54, 0x4f,
-	0x44, 0x45, 0x54, 0x45, 0x43, 0x54, 0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, 0x43, 0x4c, 0x4f, 0x55,
-	0x44, 0x10, 0x01, 0x12, 0x0f, 0x0a, 0x0b, 0x44, 0x41, 0x54, 0x41, 0x5f, 0x43, 0x45, 0x4e, 0x54,
-	0x45, 0x52, 0x10, 0x02, 0x2a, 0x87, 0x01, 0x0a, 0x14, 0x4a, 0x69, 0x72, 0x61, 0x49, 0x6e, 0x73,
-	0x74, 0x61, 0x6c, 0x6c, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x12, 0x25, 0x0a,
-	0x21, 0x4a, 0x49, 0x52, 0x41, 0x5f, 0x49, 0x4e, 0x53, 0x54, 0x41, 0x4c, 0x4c, 0x41, 0x54, 0x49,
-	0x4f, 0x4e, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x41, 0x55, 0x54, 0x4f, 0x44, 0x45, 0x54, 0x45,
-	0x43, 0x54, 0x10, 0x00, 0x12, 0x20, 0x0a, 0x1c, 0x4a, 0x49, 0x52, 0x41, 0x5f, 0x49, 0x4e, 0x53,
-	0x54, 0x41, 0x4c, 0x4c, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x43,
-	0x4c, 0x4f, 0x55, 0x44, 0x10, 0x01, 0x12, 0x26, 0x0a, 0x22, 0x4a, 0x49, 0x52, 0x41, 0x5f, 0x49,
-	0x4e, 0x53, 0x54, 0x41, 0x4c, 0x4c, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x54, 0x59, 0x50, 0x45,
-	0x5f, 0x44, 0x41, 0x54, 0x41, 0x5f, 0x43, 0x45, 0x4e, 0x54, 0x45, 0x52, 0x10, 0x02, 0x42, 0x3b,
-	0x5a, 0x39, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x72, 0x75,
-	0x66, 0x66, 0x6c, 0x65, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x2f, 0x74, 0x72, 0x75,
-	0x66, 0x66, 0x6c, 0x65, 0x68, 0x6f, 0x67, 0x2f, 0x76, 0x33, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70,
-	0x62, 0x2f, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f,
-	0x74, 0x6f, 0x33,
+	0x20, 0x01, 0x28, 0x03, 0x52, 0x05, 0x64, 0x65, 0x6c, 0x61, 0x79, 0x12, 0x18, 0x0a, 0x07, 0x74,
+	0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x03, 0x52, 0x07, 0x74, 0x69,
+	0x6d, 0x65, 0x6f, 0x75, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x75, 0x73, 0x65, 0x72, 0x5f, 0x61, 0x67,
+	0x65, 0x6e, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x75, 0x73, 0x65, 0x72, 0x41,
+	0x67, 0x65, 0x6e, 0x74, 0x12, 0x23, 0x0a, 0x0d, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x5f, 0x72,
+	0x6f, 0x62, 0x6f, 0x74, 0x73, 0x18, 0x07, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x69, 0x67, 0x6e,
+	0x6f, 0x72, 0x65, 0x52, 0x6f, 0x62, 0x6f, 0x74, 0x73, 0x2a, 0xd9, 0x09, 0x0a, 0x0a, 0x53, 0x6f,
+	0x75, 0x72, 0x63, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x1d, 0x0a, 0x19, 0x53, 0x4f, 0x55, 0x52,
+	0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x41, 0x5a, 0x55, 0x52, 0x45, 0x5f, 0x53, 0x54,
+	0x4f, 0x52, 0x41, 0x47, 0x45, 0x10, 0x00, 0x12, 0x19, 0x0a, 0x15, 0x53, 0x4f, 0x55, 0x52, 0x43,
+	0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x42, 0x49, 0x54, 0x42, 0x55, 0x43, 0x4b, 0x45, 0x54,
+	0x10, 0x01, 0x12, 0x18, 0x0a, 0x14, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50,
+	0x45, 0x5f, 0x43, 0x49, 0x52, 0x43, 0x4c, 0x45, 0x43, 0x49, 0x10, 0x02, 0x12, 0x1a, 0x0a, 0x16,
+	0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x43, 0x4f, 0x4e, 0x46,
+	0x4c, 0x55, 0x45, 0x4e, 0x43, 0x45, 0x10, 0x03, 0x12, 0x16, 0x0a, 0x12, 0x53, 0x4f, 0x55, 0x52,
+	0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x44, 0x4f, 0x43, 0x4b, 0x45, 0x52, 0x10, 0x04,
+	0x12, 0x13, 0x0a, 0x0f, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f,
+	0x45, 0x43, 0x52, 0x10, 0x05, 0x12, 0x13, 0x0a, 0x0f, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f,
+	0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x43, 0x53, 0x10, 0x06, 0x12, 0x16, 0x0a, 0x12, 0x53, 0x4f,
+	0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x49, 0x54, 0x48, 0x55, 0x42,
+	0x10, 0x07, 0x12, 0x1a, 0x0a, 0x16, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50,
+	0x45, 0x5f, 0x50, 0x55, 0x42, 0x4c, 0x49, 0x43, 0x5f, 0x47, 0x49, 0x54, 0x10, 0x08, 0x12, 0x16,
+	0x0a, 0x12, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x49,
+	0x54, 0x4c, 0x41, 0x42, 0x10, 0x09, 0x12, 0x14, 0x0a, 0x10, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45,
+	0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4a, 0x49, 0x52, 0x41, 0x10, 0x0a, 0x12, 0x24, 0x0a, 0x20,
+	0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4e, 0x50, 0x4d, 0x5f,
+	0x55, 0x4e, 0x41, 0x55, 0x54, 0x48, 0x44, 0x5f, 0x50, 0x41, 0x43, 0x4b, 0x41, 0x47, 0x45, 0x53,
+	0x10, 0x0b, 0x12, 0x25, 0x0a, 0x21, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50,
+	0x45, 0x5f, 0x50, 0x59, 0x50, 0x49, 0x5f, 0x55, 0x4e, 0x41, 0x55, 0x54, 0x48, 0x44, 0x5f, 0x50,
+	0x41, 0x43, 0x4b, 0x41, 0x47, 0x45, 0x53, 0x10, 0x0c, 0x12, 0x12, 0x0a, 0x0e, 0x53, 0x4f, 0x55,
+	0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x33, 0x10, 0x0d, 0x12, 0x15, 0x0a,
+	0x11, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x4c, 0x41,
+	0x43, 0x4b, 0x10, 0x0e, 0x12, 0x1a, 0x0a, 0x16, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54,
+	0x59, 0x50, 0x45, 0x5f, 0x46, 0x49, 0x4c, 0x45, 0x53, 0x59, 0x53, 0x54, 0x45, 0x4d, 0x10, 0x0f,
+	0x12, 0x13, 0x0a, 0x0f, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f,
+	0x47, 0x49, 0x54, 0x10, 0x10, 0x12, 0x14, 0x0a, 0x10, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f,
+	0x54, 0x59, 0x50, 0x45, 0x5f, 0x54, 0x45, 0x53, 0x54, 0x10, 0x11, 0x12, 0x1b, 0x0a, 0x17, 0x53,
+	0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x33, 0x5f, 0x55, 0x4e,
+	0x41, 0x55, 0x54, 0x48, 0x45, 0x44, 0x10, 0x12, 0x12, 0x2a, 0x0a, 0x26, 0x53, 0x4f, 0x55, 0x52,
+	0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x49, 0x54, 0x48, 0x55, 0x42, 0x5f, 0x55,
+	0x4e, 0x41, 0x55, 0x54, 0x48, 0x45, 0x4e, 0x54, 0x49, 0x43, 0x41, 0x54, 0x45, 0x44, 0x5f, 0x4f,
+	0x52, 0x47, 0x10, 0x13, 0x12, 0x19, 0x0a, 0x15, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54,
+	0x59, 0x50, 0x45, 0x5f, 0x42, 0x55, 0x49, 0x4c, 0x44, 0x4b, 0x49, 0x54, 0x45, 0x10, 0x14, 0x12,
+	0x16, 0x0a, 0x12, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47,
+	0x45, 0x52, 0x52, 0x49, 0x54, 0x10, 0x15, 0x12, 0x17, 0x0a, 0x13, 0x53, 0x4f, 0x55, 0x52, 0x43,
+	0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4a, 0x45, 0x4e, 0x4b, 0x49, 0x4e, 0x53, 0x10, 0x16,
+	0x12, 0x15, 0x0a, 0x11, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f,
+	0x54, 0x45, 0x41, 0x4d, 0x53, 0x10, 0x17, 0x12, 0x21, 0x0a, 0x1d, 0x53, 0x4f, 0x55, 0x52, 0x43,
+	0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4a, 0x46, 0x52, 0x4f, 0x47, 0x5f, 0x41, 0x52, 0x54,
+	0x49, 0x46, 0x41, 0x43, 0x54, 0x4f, 0x52, 0x59, 0x10, 0x18, 0x12, 0x16, 0x0a, 0x12, 0x53, 0x4f,
+	0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x59, 0x53, 0x4c, 0x4f, 0x47,
+	0x10, 0x19, 0x12, 0x27, 0x0a, 0x23, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50,
+	0x45, 0x5f, 0x50, 0x55, 0x42, 0x4c, 0x49, 0x43, 0x5f, 0x45, 0x56, 0x45, 0x4e, 0x54, 0x5f, 0x4d,
+	0x4f, 0x4e, 0x49, 0x54, 0x4f, 0x52, 0x49, 0x4e, 0x47, 0x10, 0x1a, 0x12, 0x1e, 0x0a, 0x1a, 0x53,
+	0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x4c, 0x41, 0x43, 0x4b,
+	0x5f, 0x52, 0x45, 0x41, 0x4c, 0x54, 0x49, 0x4d, 0x45, 0x10, 0x1b, 0x12, 0x1c, 0x0a, 0x18, 0x53,
+	0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x4f, 0x4f, 0x47, 0x4c,
+	0x45, 0x5f, 0x44, 0x52, 0x49, 0x56, 0x45, 0x10, 0x1c, 0x12, 0x1a, 0x0a, 0x16, 0x53, 0x4f, 0x55,
+	0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x48, 0x41, 0x52, 0x45, 0x50, 0x4f,
+	0x49, 0x4e, 0x54, 0x10, 0x1d, 0x12, 0x1c, 0x0a, 0x18, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f,
+	0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x43, 0x53, 0x5f, 0x55, 0x4e, 0x41, 0x55, 0x54, 0x48, 0x45,
+	0x44, 0x10, 0x1e, 0x12, 0x1b, 0x0a, 0x17, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59,
+	0x50, 0x45, 0x5f, 0x41, 0x5a, 0x55, 0x52, 0x45, 0x5f, 0x52, 0x45, 0x50, 0x4f, 0x53, 0x10, 0x1f,
+	0x12, 0x18, 0x0a, 0x14, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f,
+	0x54, 0x52, 0x41, 0x56, 0x49, 0x53, 0x43, 0x49, 0x10, 0x20, 0x12, 0x17, 0x0a, 0x13, 0x53, 0x4f,
+	0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x50, 0x4f, 0x53, 0x54, 0x4d, 0x41,
+	0x4e, 0x10, 0x21, 0x12, 0x17, 0x0a, 0x13, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59,
+	0x50, 0x45, 0x5f, 0x57, 0x45, 0x42, 0x48, 0x4f, 0x4f, 0x4b, 0x10, 0x22, 0x12, 0x1d, 0x0a, 0x19,
+	0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x45, 0x4c, 0x41, 0x53,
+	0x54, 0x49, 0x43, 0x53, 0x45, 0x41, 0x52, 0x43, 0x48, 0x10, 0x23, 0x12, 0x1b, 0x0a, 0x17, 0x53,
+	0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x48, 0x55, 0x47, 0x47, 0x49,
+	0x4e, 0x47, 0x46, 0x41, 0x43, 0x45, 0x10, 0x24, 0x12, 0x23, 0x0a, 0x1f, 0x53, 0x4f, 0x55, 0x52,
+	0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x49, 0x54, 0x48, 0x55, 0x42, 0x5f, 0x45,
+	0x58, 0x50, 0x45, 0x52, 0x49, 0x4d, 0x45, 0x4e, 0x54, 0x41, 0x4c, 0x10, 0x25, 0x12, 0x16, 0x0a,
+	0x12, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x45, 0x4e,
+	0x54, 0x52, 0x59, 0x10, 0x26, 0x12, 0x1f, 0x0a, 0x1b, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f,
+	0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x49, 0x54, 0x48, 0x55, 0x42, 0x5f, 0x52, 0x45, 0x41, 0x4c,
+	0x54, 0x49, 0x4d, 0x45, 0x10, 0x27, 0x12, 0x15, 0x0a, 0x11, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45,
+	0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x54, 0x44, 0x49, 0x4e, 0x10, 0x28, 0x12, 0x20, 0x0a,
+	0x1c, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x4c, 0x41,
+	0x43, 0x4b, 0x5f, 0x43, 0x4f, 0x4e, 0x54, 0x49, 0x4e, 0x55, 0x4f, 0x55, 0x53, 0x10, 0x29, 0x12,
+	0x1f, 0x0a, 0x1b, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4a,
+	0x53, 0x4f, 0x4e, 0x5f, 0x45, 0x4e, 0x55, 0x4d, 0x45, 0x52, 0x41, 0x54, 0x4f, 0x52, 0x10, 0x2a,
+	0x12, 0x13, 0x0a, 0x0f, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f,
+	0x57, 0x45, 0x42, 0x10, 0x2b, 0x2a, 0x47, 0x0a, 0x19, 0x42, 0x69, 0x74, 0x62, 0x75, 0x63, 0x6b,
+	0x65, 0x74, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79,
+	0x70, 0x65, 0x12, 0x0e, 0x0a, 0x0a, 0x41, 0x55, 0x54, 0x4f, 0x44, 0x45, 0x54, 0x45, 0x43, 0x54,
+	0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, 0x43, 0x4c, 0x4f, 0x55, 0x44, 0x10, 0x01, 0x12, 0x0f, 0x0a,
+	0x0b, 0x44, 0x41, 0x54, 0x41, 0x5f, 0x43, 0x45, 0x4e, 0x54, 0x45, 0x52, 0x10, 0x02, 0x2a, 0x87,
+	0x01, 0x0a, 0x14, 0x4a, 0x69, 0x72, 0x61, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x61, 0x74,
+	0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x12, 0x25, 0x0a, 0x21, 0x4a, 0x49, 0x52, 0x41, 0x5f,
+	0x49, 0x4e, 0x53, 0x54, 0x41, 0x4c, 0x4c, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x54, 0x59, 0x50,
+	0x45, 0x5f, 0x41, 0x55, 0x54, 0x4f, 0x44, 0x45, 0x54, 0x45, 0x43, 0x54, 0x10, 0x00, 0x12, 0x20,
+	0x0a, 0x1c, 0x4a, 0x49, 0x52, 0x41, 0x5f, 0x49, 0x4e, 0x53, 0x54, 0x41, 0x4c, 0x4c, 0x41, 0x54,
+	0x49, 0x4f, 0x4e, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x43, 0x4c, 0x4f, 0x55, 0x44, 0x10, 0x01,
+	0x12, 0x26, 0x0a, 0x22, 0x4a, 0x49, 0x52, 0x41, 0x5f, 0x49, 0x4e, 0x53, 0x54, 0x41, 0x4c, 0x4c,
+	0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x44, 0x41, 0x54, 0x41, 0x5f,
+	0x43, 0x45, 0x4e, 0x54, 0x45, 0x52, 0x10, 0x02, 0x42, 0x3b, 0x5a, 0x39, 0x67, 0x69, 0x74, 0x68,
+	0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73, 0x65,
+	0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68, 0x6f,
+	0x67, 0x2f, 0x76, 0x33, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x73, 0x6f, 0x75, 0x72,
+	0x63, 0x65, 0x73, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
 }
 
 var (
diff --git a/pkg/pb/sourcespb/sources.pb.validate.go b/pkg/pb/sourcespb/sources.pb.validate.go
index fab83443aa12..12252d102699 100644
--- a/pkg/pb/sourcespb/sources.pb.validate.go
+++ b/pkg/pb/sourcespb/sources.pb.validate.go
@@ -7266,6 +7266,8 @@ func (m *Web) validate(all bool) error {
 
 	// no validation rules for Delay
 
+	// no validation rules for Timeout
+
 	// no validation rules for UserAgent
 
 	// no validation rules for IgnoreRobots
diff --git a/pkg/sources/sources.go b/pkg/sources/sources.go
index f55b62b68f62..b666182810c2 100644
--- a/pkg/sources/sources.go
+++ b/pkg/sources/sources.go
@@ -484,19 +484,33 @@ type JSONEnumeratorConfig struct {
 
 // WebConfig defines the configuration for the web source.
 type WebConfig struct {
-	// URL is the list of starting points for scanning.
+	// URLs are the seed URLs to scan. At least one is required.
+	// Each URL is crawled independently with its own collector.
 	URLs []string
 
-	// Crawl determines whether to follow links from the starting page.
+	// Crawl controls whether links discovered on each page are followed.
+	// When false, only the seed URLs themselves are scanned.
 	Crawl bool
 
-	// Depth controls how many link hops to follow when Crawl is true.
-	// 0 = only the starting URL, 1 = starting URL and direct links, etc.
+	// Depth is the maximum number of link hops to follow when Crawl is true.
+	// 1 = seed + direct links; 2 = one level deeper; 0 = unlimited.
 	Depth int
 
-	// Delay is the delay (in seconds) between requests to the same domain.
-	Delay        int
-	UserAgent    string
+	// Delay is the number of seconds to wait between requests to the same
+	// domain. Increase this to reduce load on the target server.
+	Delay int
+
+	// Timeout is the maximum number of seconds to spend crawling a single
+	// seed URL before aborting. Applied independently per URL.
+	// Defaults to 30 seconds if unset or zero.
+	Timeout int
+
+	// UserAgent is the User-Agent header sent with each request.
+	// Defaults to a TruffleHog identifier if empty.
+	UserAgent string
+
+	// IgnoreRobots disables robots.txt enforcement when true.
+	// Only enable this if you have explicit permission to crawl the target site.
 	IgnoreRobots bool
 }
 
diff --git a/pkg/sources/web/web.go b/pkg/sources/web/web.go
index 35b1b3d5f470..bcdb69db8a2a 100644
--- a/pkg/sources/web/web.go
+++ b/pkg/sources/web/web.go
@@ -6,11 +6,11 @@ import (
 	"fmt"
 	"net/url"
 	"strings"
-	"sync"
 	"time"
 
 	"github.com/gocolly/colly/v2"
 	"golang.org/x/net/html"
+	"golang.org/x/sync/errgroup"
 	"google.golang.org/protobuf/proto"
 	"google.golang.org/protobuf/types/known/anypb"
 
@@ -68,6 +68,11 @@ func (s *Source) Init(ctx context.Context, name string, jobId sources.JobID, sou
 		s.conn.UserAgent = "trufflehog-web (+https://github.com/trufflesecurity/trufflehog)"
 	}
 
+	// The 30-second timeout is a safety net
+	if s.conn.GetTimeout() <= 0 {
+		s.conn.Timeout = 30
+	}
+
 	if s.conn.GetIgnoreRobots() {
 		ctx.Logger().Info("Warning: Robots.txt is ignored. Only use this if you have permission to crawl the target site.")
 	}
@@ -76,10 +81,6 @@ func (s *Source) Init(ctx context.Context, name string, jobId sources.JobID, sou
 	if len(s.conn.GetUrls()) == 0 {
 		return errors.New("no URL provided")
 	}
-	// TODO: Enable support for more than one URLs
-	if len(s.conn.GetUrls()) > 1 {
-		return errors.New("only one base URL is allowed right now")
-	}
 
 	// Validate URLs format
 	for _, u := range s.conn.GetUrls() {
@@ -97,26 +98,28 @@ func (s *Source) Init(ctx context.Context, name string, jobId sources.JobID, sou
 
 // Chunks emits data over a channel that is decoded and scanned for secrets.
 func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ ...sources.ChunkingTarget) error {
-	var wg sync.WaitGroup
 	jobIDStr := fmt.Sprint(s.jobId)
 
-	// Create a background context for crawling (independent of incoming ctx)
-	crawlCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	// Create a background context for crawling (independent of incoming ctx).
+	crawlCtx, cancel := context.WithTimeout(context.Background(), time.Duration(s.conn.GetTimeout())*time.Second)
 	defer cancel()
 
-	for _, url := range s.conn.GetUrls() {
-		ctx.Logger().V(5).Info("Processing Url", "url", url)
-		wg.Add(1)
-		go func(url string) {
-			defer wg.Done()
-			s.crawlURL(crawlCtx, url, chunksChan)
-		}(url)
+	eg, _ := errgroup.WithContext(crawlCtx)
 
+	for _, u := range s.conn.GetUrls() {
+		u := u // capture
+		ctx.Logger().V(5).Info("Processing Url", "url", u)
 		webUrlsScanned.WithLabelValues(s.name, jobIDStr).Inc()
+		eg.Go(func() error {
+			return s.crawlURL(crawlCtx, u, chunksChan)
+		})
+	}
+
+	if err := eg.Wait(); err != nil {
+		ctx.Logger().Error(err, "One or more crawls failed")
+		return err
 	}
 
-	// Block until all crawls complete
-	wg.Wait()
 	ctx.Logger().Info("All crawls completed")
 	return nil
 }
@@ -129,19 +132,25 @@ func (s *Source) crawlURL(ctx context.Context, seedURL string, chunksChan chan *
 		"ignore_robots", s.conn.GetIgnoreRobots(),
 	)
 
-	url, err := url.Parse(seedURL)
+	parsedURL, err := url.Parse(seedURL)
 	if err != nil {
 		return fmt.Errorf("invalid URL %q: %w", seedURL, err)
 	}
 
+	// docs: http://go-colly.org/docs/introduction/configuration/
 	collector := colly.NewCollector(
 		colly.UserAgent(s.conn.GetUserAgent()),
-		colly.AllowedDomains(url.Hostname(), fmt.Sprintf("*.%s", url.Hostname())), // with subdomains
+		colly.AllowedDomains(parsedURL.Hostname(), fmt.Sprintf("*.%s", parsedURL.Hostname())), // with subdomains
 		colly.Async(true),
 	)
 
+	// Apply depth limit only when crawling is enabled and a positive depth is set.
+	if s.conn.GetCrawl() && s.conn.GetDepth() > 0 {
+		collector.MaxDepth = int(s.conn.GetDepth())
+	}
+
 	// By default, the crawler respects robots.txt rules. Setting IgnoreRobotsTxt to true overrides this behavior.
-	// Users can enable this only when they have explicit permission to crawl the site.
+	// Users can enable this only when they have explicit permission to crawl the target site.
 	collector.IgnoreRobotsTxt = s.conn.GetIgnoreRobots()
 
 	collector.Limit(&colly.LimitRule{
@@ -152,7 +161,7 @@ func (s *Source) crawlURL(ctx context.Context, seedURL string, chunksChan chan *
 
 	// Set up callbacks
 	collector.OnResponse(func(r *colly.Response) {
-		ctx.Logger().Info("Response recieved")
+		ctx.Logger().Info("Response received")
 		if err := s.processChunk(ctx, r, chunksChan); err != nil {
 			ctx.Logger().Error(err, "error processing page")
 		}
@@ -161,6 +170,36 @@ func (s *Source) crawlURL(ctx context.Context, seedURL string, chunksChan chan *
 		ctx.Logger().Error(err, "error fetching page", "url", r.Request.URL)
 	})
 
+	// Follow links only when crawling is explicitly enabled.
+	if s.conn.GetCrawl() {
+		collector.OnHTML("a[href]", func(e *colly.HTMLElement) {
+			link := e.Request.AbsoluteURL(e.Attr("href"))
+			if link == "" {
+				return
+			}
+
+			if err := e.Request.Visit(link); err != nil {
+				if _, ok := err.(*colly.AlreadyVisitedError); !ok {
+					ctx.Logger().V(5).Info("Skipping link", "url", link, "reason", err)
+				}
+			}
+		})
+
+		// Also enqueue linked JavaScript files - a common location for hardcoded secrets.
+		collector.OnHTML("script[src]", func(e *colly.HTMLElement) {
+			src := e.Request.AbsoluteURL(e.Attr("src"))
+			if src == "" {
+				return
+			}
+
+			if err := e.Request.Visit(src); err != nil {
+				if _, ok := err.(*colly.AlreadyVisitedError); !ok {
+					ctx.Logger().V(5).Info("Skipping script", "url", src, "reason", err)
+				}
+			}
+		})
+	}
+
 	// Create a channel to signal when the crawl is done.
 	done := make(chan struct{})
 	go func() {
diff --git a/pkg/sources/web/web_test.go b/pkg/sources/web/web_test.go
index 8ed839f8bdcb..07ae70be2468 100644
--- a/pkg/sources/web/web_test.go
+++ b/pkg/sources/web/web_test.go
@@ -1,12 +1,15 @@
 package web
 
 import (
+	"fmt"
 	"net/http"
 	"net/http/httptest"
 	"sync"
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 	"google.golang.org/protobuf/types/known/anypb"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
@@ -15,62 +18,414 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
 )
 
-func TestWebSource_HappyPath(t *testing.T) {
-	// Create a test server that returns a simple HTML page.
-	testServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		w.Header().Set("Content-Type", "text/html; charset=utf-8")
-		w.Write([]byte(`<!DOCTYPE html><html><head><title>Test Page</title></head><body>Hello, world!</body></html>`))
-	}))
-	defer testServer.Close()
-
-	// Build the web source configuration.
-	webConfig := &sourcespb.Web{
-		Urls:  []string{testServer.URL},
-		Crawl: false,
-		Depth: 0,
-		Delay: 0,
-	}
-
+// helper: marshal a *sourcespb.Web config into an *anypb.Any and Init a Source.
+func initSource(t *testing.T, cfg *sourcespb.Web, concurrency int) *Source {
+	t.Helper()
 	conn := &anypb.Any{}
-	err := conn.MarshalFrom(webConfig)
-	assert.NoError(t, err)
-
+	require.NoError(t, conn.MarshalFrom(cfg))
 	s := &Source{}
-	err = s.Init(context.TODO(), "test source", 0, 0, false, conn, 1)
-	assert.NoError(t, err)
+	require.NoError(t, s.Init(context.TODO(), "test-source", 0, 0, false, conn, concurrency))
+	return s
+}
 
+// helper: run Chunks and collect all emitted chunks.
+func collectChunks(t *testing.T, s *Source) []*sources.Chunk {
+	t.Helper()
+	chunksChan := make(chan *sources.Chunk, 16)
 	var wg sync.WaitGroup
-	chunksChan := make(chan *sources.Chunk, 1)
-	chunkCounter := 0
-
-	// Collect all chunks.
 	var chunks []*sources.Chunk
 	wg.Add(1)
 	go func() {
 		defer wg.Done()
-		for chunk := range chunksChan {
-			assert.NotEmpty(t, chunk)
-			chunkCounter++
-			chunks = append(chunks, chunk)
+		for c := range chunksChan {
+			chunks = append(chunks, c)
 		}
 	}()
-
-	err = s.Chunks(context.TODO(), chunksChan)
-	assert.NoError(t, err)
-
+	require.NoError(t, s.Chunks(context.TODO(), chunksChan))
 	close(chunksChan)
 	wg.Wait()
+	return chunks
+}
+
+// Init validation
+
+func TestInit_NoURL(t *testing.T) {
+	conn := &anypb.Any{}
+	require.NoError(t, conn.MarshalFrom(&sourcespb.Web{}))
+	s := &Source{}
+	err := s.Init(context.TODO(), "test", 0, 0, false, conn, 1)
+	assert.Error(t, err)
+}
+
+func TestInit_DefaultUserAgent(t *testing.T) {
+	conn := &anypb.Any{}
+	require.NoError(t, conn.MarshalFrom(&sourcespb.Web{
+		Urls: []string{"http://example.com"},
+	}))
+	s := &Source{}
+	require.NoError(t, s.Init(context.TODO(), "test", 0, 0, false, conn, 1))
+	assert.Contains(t, s.conn.GetUserAgent(), "trufflehog")
+}
+
+func TestInit_CustomUserAgent(t *testing.T) {
+	conn := &anypb.Any{}
+	require.NoError(t, conn.MarshalFrom(&sourcespb.Web{
+		Urls:      []string{"http://example.com"},
+		UserAgent: "my-bot/1.0",
+	}))
+	s := &Source{}
+	require.NoError(t, s.Init(context.TODO(), "test", 0, 0, false, conn, 1))
+	assert.Equal(t, "my-bot/1.0", s.conn.GetUserAgent())
+}
+
+func TestInit_ZeroConcurrencyDefaultsToOne(t *testing.T) {
+	conn := &anypb.Any{}
+	require.NoError(t, conn.MarshalFrom(&sourcespb.Web{
+		Urls: []string{"http://example.com"},
+	}))
+	s := &Source{}
+	require.NoError(t, s.Init(context.TODO(), "test", 0, 0, false, conn, 0))
+	assert.Equal(t, 1, s.concurrency)
+}
+
+// Happy path
+
+func TestChunks_HappyPath(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "text/html; charset=utf-8")
+		fmt.Fprint(w, `<!DOCTYPE html><html><head><title>Test Page</title></head><body>Hello, world!</body></html>`)
+	}))
+	defer srv.Close()
+
+	s := initSource(t, &sourcespb.Web{Urls: []string{srv.URL}}, 1)
+	chunks := collectChunks(t, s)
 
-	assert.Equal(t, 1, chunkCounter)
+	require.Equal(t, 1, len(chunks))
 	chunk := chunks[0]
 
-	// Check the chunk data.
 	assert.Contains(t, string(chunk.Data), "Hello, world!")
-	// Verify the metadata.
+
 	meta, ok := chunk.SourceMetadata.Data.(*source_metadatapb.MetaData_Web)
-	assert.True(t, ok, "expected web metadata")
+	require.True(t, ok, "expected web metadata")
 	assert.Equal(t, "Test Page", meta.Web.PageTitle)
 	assert.Equal(t, "text/html; charset=utf-8", meta.Web.ContentType)
-	assert.Equal(t, int64(1), meta.Web.Depth) // default 1 depth
+	assert.NotEmpty(t, meta.Web.Url)
 	assert.NotEmpty(t, meta.Web.Timestamp)
 }
+
+func TestChunks_TimestampIsRFC3339(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		fmt.Fprint(w, `<html><head><title>T</title></head><body>body</body></html>`)
+	}))
+	defer srv.Close()
+
+	s := initSource(t, &sourcespb.Web{Urls: []string{srv.URL}}, 1)
+	chunks := collectChunks(t, s)
+	require.Equal(t, 1, len(chunks))
+
+	meta := chunks[0].SourceMetadata.Data.(*source_metadatapb.MetaData_Web)
+	_, err := time.Parse(time.RFC3339, meta.Web.Timestamp)
+	assert.NoError(t, err, "timestamp must be valid RFC3339")
+}
+
+// Page title extraction
+
+func TestExtractPageTitle_Normal(t *testing.T) {
+	body := []byte(`<html><head><title>  Hello World  </title></head><body></body></html>`)
+	assert.Equal(t, "Hello World", extractPageTitle(body))
+}
+
+func TestExtractPageTitle_Missing(t *testing.T) {
+	body := []byte(`<html><head></head><body>no title</body></html>`)
+	assert.Equal(t, "", extractPageTitle(body))
+}
+
+func TestExtractPageTitle_Empty(t *testing.T) {
+	assert.Equal(t, "", extractPageTitle([]byte{}))
+}
+
+func TestExtractPageTitle_MalformedHTML(t *testing.T) {
+	// html.Parse is lenient; just confirm it doesn't panic and returns something sensible.
+	body := []byte(`<title>Partial`)
+	title := extractPageTitle(body)
+	// Go's html.Parse fills in missing closing tags, so the title is still parsed.
+	assert.Equal(t, "Partial", title)
+}
+
+func TestExtractPageTitle_NonHTML(t *testing.T) {
+	// Binary / JSON bodies must not panic.
+	body := []byte(`{"secret": "abc123"}`)
+	_ = extractPageTitle(body) // just confirm no panic
+}
+
+// Depth / crawl behaviour
+
+// TestChunks_NoCrawl confirms that when Crawl=false only the seed page is
+// fetched, even if the page contains an internal link.
+func TestChunks_NoCrawl(t *testing.T) {
+	mux := http.NewServeMux()
+	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "text/html")
+		fmt.Fprint(w, `<html><head><title>Root</title></head><body>
+			<a href="/page2">page 2</a>
+		</body></html>`)
+	})
+	mux.HandleFunc("/page2", func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "text/html")
+		fmt.Fprint(w, `<html><head><title>Page 2</title></head><body>Secret</body></html>`)
+	})
+	srv := httptest.NewServer(mux)
+	defer srv.Close()
+
+	// Crawl=false - only the seed URL is visited.
+	s := initSource(t, &sourcespb.Web{Urls: []string{srv.URL}, Crawl: false}, 1)
+	chunks := collectChunks(t, s)
+
+	assert.Equal(t, 1, len(chunks), "expected only the root page when Crawl=false")
+	meta := chunks[0].SourceMetadata.Data.(*source_metadatapb.MetaData_Web)
+	assert.Equal(t, "Root", meta.Web.PageTitle)
+}
+
+// TestChunks_CrawlDepth1 confirms that with Crawl=true and Depth=1 the crawler
+// visits the seed page and its direct links, but not grandchildren.
+func TestChunks_CrawlDepth1(t *testing.T) {
+	mux := http.NewServeMux()
+	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "text/html")
+		fmt.Fprint(w, `<html><head><title>Root</title></head><body>
+			<a href="/child">child</a>
+		</body></html>`)
+	})
+	mux.HandleFunc("/child", func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "text/html")
+		fmt.Fprint(w, `<html><head><title>Child</title></head><body>
+			<a href="/grandchild">grandchild</a>
+		</body></html>`)
+	})
+	mux.HandleFunc("/grandchild", func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "text/html")
+		fmt.Fprint(w, `<html><head><title>Grandchild</title></head><body>deep</body></html>`)
+	})
+	srv := httptest.NewServer(mux)
+	defer srv.Close()
+
+	s := initSource(t, &sourcespb.Web{Urls: []string{srv.URL}, Crawl: true, Depth: 2}, 1)
+	chunks := collectChunks(t, s)
+
+	titles := make(map[string]bool)
+	for _, c := range chunks {
+		meta := c.SourceMetadata.Data.(*source_metadatapb.MetaData_Web)
+		titles[meta.Web.PageTitle] = true
+	}
+
+	assert.True(t, titles["Root"], "root page should be crawled")
+	assert.True(t, titles["Child"], "child page should be crawled at depth 1")
+	assert.False(t, titles["Grandchild"], "grandchild should NOT be crawled at depth 1")
+}
+
+// TestChunks_CrawlDepth2 confirms depth-2 traversal reaches grandchildren.
+func TestChunks_CrawlDepth2(t *testing.T) {
+	mux := http.NewServeMux()
+	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "text/html")
+		fmt.Fprint(w, `<html><head><title>Root</title></head><body>
+			<a href="/child">child</a>
+		</body></html>`)
+	})
+	mux.HandleFunc("/child", func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "text/html")
+		fmt.Fprint(w, `<html><head><title>Child</title></head><body>
+			<a href="/grandchild">grandchild</a>
+		</body></html>`)
+	})
+	mux.HandleFunc("/grandchild", func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "text/html")
+		fmt.Fprint(w, `<html><head><title>Grandchild</title></head><body>deep</body></html>`)
+	})
+	srv := httptest.NewServer(mux)
+	defer srv.Close()
+
+	s := initSource(t, &sourcespb.Web{Urls: []string{srv.URL}, Crawl: true, Depth: 3}, 1)
+	chunks := collectChunks(t, s)
+
+	titles := make(map[string]bool)
+	for _, c := range chunks {
+		meta := c.SourceMetadata.Data.(*source_metadatapb.MetaData_Web)
+		titles[meta.Web.PageTitle] = true
+	}
+
+	assert.True(t, titles["Root"])
+	assert.True(t, titles["Child"])
+	assert.True(t, titles["Grandchild"])
+}
+
+// TestChunks_CrossDomainLinksIgnored ensures that links pointing to a different
+// hostname are not followed. httptest.NewServer always binds to 127.0.0.1, so
+// we cannot spin up a second "external" server on a different IP in a portable
+// way. Instead we embed a link to an unreachable external hostname directly in
+// the page HTML and assert that no chunk with that hostname is ever produced.
+func TestChunks_CrossDomainLinksIgnored(t *testing.T) {
+	const externalURL = "http://external.example.invalid/secret"
+
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "text/html")
+		fmt.Fprintf(w, `<html><head><title>Seed</title></head><body>
+			<a href="%s">external link</a>
+		</body></html>`, externalURL)
+	}))
+	defer srv.Close()
+
+	s := initSource(t, &sourcespb.Web{Urls: []string{srv.URL}, Crawl: true, Depth: 1}, 1)
+	chunks := collectChunks(t, s)
+
+	for _, c := range chunks {
+		meta := c.SourceMetadata.Data.(*source_metadatapb.MetaData_Web)
+		assert.NotContains(t, meta.Web.Url, "external.example.invalid",
+			"cross-domain URL must not appear in chunks")
+	}
+}
+
+// Content types
+
+func TestChunks_PlainTextPage(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "text/plain")
+		fmt.Fprint(w, "API_KEY=supersecret")
+	}))
+	defer srv.Close()
+
+	s := initSource(t, &sourcespb.Web{Urls: []string{srv.URL}}, 1)
+	chunks := collectChunks(t, s)
+
+	require.Equal(t, 1, len(chunks))
+	assert.Contains(t, string(chunks[0].Data), "supersecret")
+
+	meta := chunks[0].SourceMetadata.Data.(*source_metadatapb.MetaData_Web)
+	assert.Equal(t, "text/plain", meta.Web.ContentType)
+}
+
+func TestChunks_JSONPage(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "application/json")
+		fmt.Fprint(w, `{"token":"ghp_supersecret"}`)
+	}))
+	defer srv.Close()
+
+	s := initSource(t, &sourcespb.Web{Urls: []string{srv.URL}}, 1)
+	chunks := collectChunks(t, s)
+
+	require.Equal(t, 1, len(chunks))
+	assert.Contains(t, string(chunks[0].Data), "ghp_supersecret")
+}
+
+// Error / edge cases
+
+func TestChunks_ServerReturns404(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		http.NotFound(w, r)
+	}))
+	defer srv.Close()
+
+	s := initSource(t, &sourcespb.Web{Urls: []string{srv.URL}}, 1)
+	// colly treats 4xx as errors and fires OnError; no chunk should be emitted.
+	chunks := collectChunks(t, s)
+	assert.Empty(t, chunks, "a 404 response should not produce a chunk")
+}
+
+func TestChunks_EmptyBody(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "text/html")
+		// no body written
+	}))
+	defer srv.Close()
+
+	s := initSource(t, &sourcespb.Web{Urls: []string{srv.URL}}, 1)
+	chunks := collectChunks(t, s)
+	// An empty but 200-OK response should still produce a chunk (body == "").
+	require.Equal(t, 1, len(chunks))
+	assert.Empty(t, chunks[0].Data)
+}
+
+// Duplicate link deduplication
+
+// TestChunks_DuplicateLinksVisitedOnce ensures that the same URL appearing
+// multiple times in a page is only fetched once.
+func TestChunks_DuplicateLinksVisitedOnce(t *testing.T) {
+	hitCount := 0
+	mux := http.NewServeMux()
+	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "text/html")
+		fmt.Fprint(w, `<html><head><title>Root</title></head><body>
+			<a href="/dup">link1</a>
+			<a href="/dup">link2</a>
+			<a href="/dup">link3</a>
+		</body></html>`)
+	})
+	mux.HandleFunc("/dup", func(w http.ResponseWriter, r *http.Request) {
+		hitCount++
+		fmt.Fprint(w, `<html><head><title>Dup</title></head><body>dup</body></html>`)
+	})
+	srv := httptest.NewServer(mux)
+	defer srv.Close()
+
+	s := initSource(t, &sourcespb.Web{Urls: []string{srv.URL}, Crawl: true, Depth: 0}, 1)
+	_ = collectChunks(t, s)
+
+	assert.Equal(t, 1, hitCount, "duplicate links should only be fetched once")
+}
+
+// Robots.txt
+
+func TestChunks_RobotsTxtRespected(t *testing.T) {
+	mux := http.NewServeMux()
+	mux.HandleFunc("/robots.txt", func(w http.ResponseWriter, r *http.Request) {
+		fmt.Fprint(w, "User-agent: *\nDisallow: /secret\n")
+	})
+	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "text/html")
+		fmt.Fprint(w, `<html><head><title>Root</title></head><body>
+			<a href="/secret">secret</a>
+		</body></html>`)
+	})
+	secretVisited := false
+	mux.HandleFunc("/secret", func(w http.ResponseWriter, r *http.Request) {
+		secretVisited = true
+		fmt.Fprint(w, `<html><head><title>Secret</title></head><body>private</body></html>`)
+	})
+	srv := httptest.NewServer(mux)
+	defer srv.Close()
+
+	s := initSource(t, &sourcespb.Web{
+		Urls: []string{srv.URL}, Crawl: true, Depth: 1, IgnoreRobots: false,
+	}, 1)
+	_ = collectChunks(t, s)
+
+	assert.False(t, secretVisited, "/secret must not be crawled when disallowed by robots.txt")
+}
+
+func TestChunks_IgnoreRobotsTxt(t *testing.T) {
+	mux := http.NewServeMux()
+	mux.HandleFunc("/robots.txt", func(w http.ResponseWriter, r *http.Request) {
+		fmt.Fprint(w, "User-agent: *\nDisallow: /secret\n")
+	})
+	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "text/html")
+		fmt.Fprint(w, `<html><head><title>Root</title></head><body>
+			<a href="/secret">secret</a>
+		</body></html>`)
+	})
+	secretVisited := false
+	mux.HandleFunc("/secret", func(w http.ResponseWriter, r *http.Request) {
+		secretVisited = true
+		fmt.Fprint(w, `<html><head><title>Secret</title></head><body>private</body></html>`)
+	})
+	srv := httptest.NewServer(mux)
+	defer srv.Close()
+
+	s := initSource(t, &sourcespb.Web{
+		Urls: []string{srv.URL}, Crawl: true, Depth: 2, IgnoreRobots: true,
+	}, 1)
+	_ = collectChunks(t, s)
+
+	assert.True(t, secretVisited, "/secret should be crawled when IgnoreRobots=true")
+}
diff --git a/proto/sources.proto b/proto/sources.proto
index 18f28aacf5c6..ce2f28faf3e2 100644
--- a/proto/sources.proto
+++ b/proto/sources.proto
@@ -561,6 +561,7 @@ message Web {
   bool crawl = 2;
   int64 depth = 3;
   int64 delay = 4;
-  string user_agent = 5;
-  bool ignore_robots = 6;
+  int64 timeout = 5;
+  string user_agent = 6;
+  bool ignore_robots = 7;
 }

From f8d7f4ceec9caf8a80ba5b3241dcc4de8ffcc5f7 Mon Sep 17 00:00:00 2001
From: Kashif Khan <kashif.khan@trufflesec.com>
Date: Mon, 30 Mar 2026 16:23:49 +0500
Subject: [PATCH 09/20] fixed some comments

---
 main.go                |  2 +-
 pkg/sources/sources.go |  2 +-
 pkg/sources/web/web.go | 18 ++++++++++++++++++
 3 files changed, 20 insertions(+), 2 deletions(-)

diff --git a/main.go b/main.go
index 9fad356824c0..106501e87fc8 100644
--- a/main.go
+++ b/main.go
@@ -279,7 +279,7 @@ var (
 	webScan         = cli.Command("web", "Scan websites for leaked credentials.")
 	webUrls         = webScan.Flag("url", "URL to scan. Repeat the flag for multiple targets, e.g. --url https://a.com --url https://b.com. Supports http:// and https://.").Required().Strings()
 	webCrawl        = webScan.Flag("crawl", "Follow links found on each page. Without this flag only the seed URL(s) are scanned.").Default("false").Bool()
-	webDepth        = webScan.Flag("depth", "Maximum link depth to follow when --crawl is enabled. 1 = seed + direct links; 2 = one level deeper; 0 = unlimited.").Default("1").Int()
+	webDepth        = webScan.Flag("depth", "Maximum link depth to follow when --crawl is enabled. 1 = seed; 2 = one level deeper; 0 = unlimited.").Default("1").Int()
 	webDelay        = webScan.Flag("delay", "Seconds to wait between requests to the same domain. Increase this to reduce load on the target server.").Default("1").Int()
 	webTimeout      = webScan.Flag("timeout", "Seconds to spend crawling each URL before aborting. Applied per URL when multiple --url flags are given.").Default("30").Int()
 	webUserAgent    = webScan.Flag("user-agent", "User-Agent header to send with each request. Defaults to a TruffleHog identifier if not set.").String()
diff --git a/pkg/sources/sources.go b/pkg/sources/sources.go
index b666182810c2..b5ba5f03099c 100644
--- a/pkg/sources/sources.go
+++ b/pkg/sources/sources.go
@@ -493,7 +493,7 @@ type WebConfig struct {
 	Crawl bool
 
 	// Depth is the maximum number of link hops to follow when Crawl is true.
-	// 1 = seed + direct links; 2 = one level deeper; 0 = unlimited.
+	// 1 = seed; 2 = one level deeper; 0 = unlimited.
 	Depth int
 
 	// Delay is the number of seconds to wait between requests to the same
diff --git a/pkg/sources/web/web.go b/pkg/sources/web/web.go
index bcdb69db8a2a..87937b1d6088 100644
--- a/pkg/sources/web/web.go
+++ b/pkg/sources/web/web.go
@@ -252,18 +252,36 @@ func (s *Source) processChunk(ctx context.Context, data *colly.Response, chunksC
 	return common.CancellableWrite(ctx, chunksChan, chunk)
 }
 
+// extractPageTitle parses an HTML document and returns the text content of the
+// first <title> element, with leading and trailing whitespace trimmed.
+// Returns an empty string if the body is empty, cannot be parsed, or contains
+// no <title> element.
 func extractPageTitle(body []byte) string {
 	doc, err := html.Parse(bytes.NewReader(body))
 	if err != nil {
 		return ""
 	}
+
 	var title string
+
+	// f is a recursive depth-first walker over the HTML node tree.
+	// It is declared as a variable first so that the closure can reference
+	// itself when recursing into child nodes.
 	var f func(*html.Node)
 	f = func(n *html.Node) {
+		if title != "" {
+			return // already found, skip the rest of the tree
+		}
+		// We are only interested in element nodes (e.g. <title>, <div>).
+		// Text, comment, and doctype nodes are skipped by this check.
 		if n.Type == html.ElementNode && n.Data == "title" && n.FirstChild != nil {
+			// <title> content is always a single text node directly inside
+			// the element. n.FirstChild.Data holds the raw string value.
 			title = strings.TrimSpace(n.FirstChild.Data)
 			return
 		}
+
+		// Recurse into child nodes to continue the depth-first traversal.
 		for c := n.FirstChild; c != nil; c = c.NextSibling {
 			f(c)
 		}

From a94c20f84a673afe3e2e7477459ed6f309b55119 Mon Sep 17 00:00:00 2001
From: Kashif Khan <kashif.khan@trufflesec.com>
Date: Mon, 30 Mar 2026 16:28:53 +0500
Subject: [PATCH 10/20] updated README.md

---
 pkg/sources/web/README.md | 227 +++++++-------------------------------
 1 file changed, 38 insertions(+), 189 deletions(-)

diff --git a/pkg/sources/web/README.md b/pkg/sources/web/README.md
index 0f32075c98e8..987c68f099e7 100644
--- a/pkg/sources/web/README.md
+++ b/pkg/sources/web/README.md
@@ -1,176 +1,80 @@
 # TruffleHog Web Source
 
-The Web source enables TruffleHog to crawl and scan websites for secrets and sensitive information. It uses the Colly web scraper framework to systematically browse web pages and analyze their content for exposed credentials, API keys, private keys, and other secrets.
-
-## Features
-
-- **Web Crawling**: Automatically crawl websites starting from a seed URL
-- **Robots.txt Compliance**: Respects website `robots.txt` rules for ethical crawling
-- **Subdomain Support**: Crawls subdomains of the target domain
-- **Customizable Delays**: Set delays between requests to avoid overwhelming servers
-- **Metadata Extraction**: Captures page titles, URLs, content types, and timestamps
-- **Error Handling**: Gracefully handles network errors and HTTP failures
+Crawls and scans websites for secrets and sensitive information using the [Colly](http://go-colly.org/) web scraping framework.
 
 ## Configuration
 
-### Required Parameters
-
-- **`--url`**: One or more URLs to scan (required)
-  - Supports both `http://` and `https://` URLs
-  - Examples: `https://example.com` or `http://staging.app.com`
-  - Can specify multiple URLs: `--url https://example.com --url https://app.com`
-
-### Optional Parameters
+### Required
 
-- **`--crawl`**: Enable crawling to follow links discovered on pages (default: `false`)
-  - `false`: Only scan the provided seed URL(s), don't follow links
-  - `true`: Follow discovered links to scan additional pages
-  - Useful for comprehensive scanning of entire websites
+| Flag | Description |
+|------|-------------|
+| `--url` | URL to scan. Repeat for multiple targets: `--url https://a.com --url https://b.com`. Supports `http://` and `https://`. |
 
-- **`--depth`**: Maximum link depth to follow when crawling (default: `1`)
-  - `0`: Only scan the seed URL(s), no link following
-  - `1`: Scan seed URL(s) + direct links from those pages
-  - `2`: Scan seed + direct links + links from those pages (two levels deep)
-  - `3+`: Continue following links up to the specified depth
-  - Note: Deeper scans take longer and consume more resources
+### Optional
 
-- **`--delay`**: Delay in seconds between requests to the same domain (default: `1`)
-  - Recommended: 1-2 seconds for responsible, server-friendly scanning
-  - Helps avoid overwhelming the target website
-  - Respects `robots.txt` Crawl-delay directives when present
+| Flag | Default | Description |
+|------|---------|-------------|
+| `--crawl` | `false` | Follow links found on each page. Without this flag only the seed URL(s) are scanned. |
+| `--depth` | `1` | Maximum link depth to follow when `--crawl` is enabled. `1` = seed; `2` = one level deeper; `0` = unlimited. Has no effect without `--crawl`. |
+| `--delay` | `1` | Seconds to wait between requests to the same domain. Increase this to reduce load on the target server. |
+| `--timeout` | `30` | Seconds to spend crawling each URL before aborting. Applied independently per URL. |
+| `--user-agent` | TruffleHog identifier | User-Agent header sent with each request. |
+| `--ignore-robots` | `false` | Ignore `robots.txt` restrictions. Only enable this if you have explicit permission to crawl the target site. |
 
 ## Usage
 
-### Command Line Examples
-
-**Scan single URL only (no crawling)**
+**Scan a single page (no crawling)**
 ```bash
 trufflehog web --url https://example.com
 ```
 
-**Scan single URL with 1 level of link following**
+**Scan a page and its direct links**
 ```bash
-trufflehog web --url https://example.com --crawl --depth 1 --delay 2
+trufflehog web --url https://example.com --crawl --depth 2 --delay 2
 ```
 
-**Scan multiple URLs with deeper crawling**
+**Scan multiple URLs two levels deep**
 ```bash
 trufflehog web \
   --url https://example.com \
   --url https://app.example.com \
   --crawl \
-  --depth 2 \
+  --depth 3 \
   --delay 1
 ```
 
-**Comprehensive website scan (2 levels deep, 2-second delays)**
+**Scan with a short per-URL timeout**
 ```bash
-trufflehog web --url https://mycompany.com --crawl --depth 2 --delay 2
+trufflehog web --url https://example.com --crawl --depth 2 --timeout 60
 ```
 
 ## Behavior
 
-### Domain Handling
-
-- Crawls the exact domain provided (e.g., `example.com`)
-- Crawls all subdomains (e.g., `www.example.com`, `mail.example.com`)
-- Does NOT crawl other domains or external links
-
-### Robots.txt Respect
-
-By default, the crawler respects `robots.txt` files:
-- Reads `robots.txt` from the website root
-- Skips paths marked as disallowed
-- Honors crawl-delay directives
-
-To ignore `robots.txt` (not recommended), modify the code:
-```go
-collector.IgnoreRobotsTxt = true
-```
-
-### User Agent
-
-The crawler identifies itself as:
-```
-trufflehog-web (+https://github.com/trufflesecurity/trufflehog)
-```
-
-This allows website administrators to identify TruffleHog requests in their logs.
-
-## Example Scenarios
-
-### Quick Scan - Check Single Page for Secrets
-
-```bash
-trufflehog web --url https://mycompany.com
-```
-- Scans only the homepage
-- No link following
-- 1 second delay between any requests
-
-### Thorough Scan - Crawl Entire Website
-
-```bash
-trufflehog web \
-  --url https://mycompany.com \
-  --crawl \
-  --depth 2 \
-  --delay 2
-```
-- Starts from homepage
-- Follows links up to 2 levels deep
-- Respectful 2-second delays between requests
-
-### Multi-Site Scan - Check Multiple URLs
-
-```bash
-trufflehog web \
-  --url https://main.company.com \
-  --url https://staging.company.com \
-  --url https://api.company.com \
-  --crawl \
-  --depth 1 \
-  --delay 1
-```
-- Scans 3 different URLs
-- Follows direct links from each
-- 1 second delay to keep scanning fast
+### Domain scope
 
-## Best Practices
+The crawler visits the exact domain provided and all of its subdomains. External links to other domains are always skipped.
 
-1. **Always Get Permission**: Only scan websites you own or have explicit permission to scan
+### Depth counting
 
-2. **Start Conservative**: Begin with no crawling, then gradually increase depth if needed
+Depth is counted in hops from the seed URL. The seed itself is hop 1; pages linked directly from it are hop 2, and so on. Setting `--depth 0` with `--crawl` enables unlimited traversal.
 
-3. **Use Appropriate Delays**: 
-   - `--delay 1`: Good for most websites
-   - `--delay 2`: Large or busy websites
-   - `--delay 0.5`: Staging/internal websites only
+### Robots.txt
 
-4. **Respect Crawl Depth**: 
-   - `--depth 0`: Just the seed URL (fastest, least coverage)
-   - `--depth 1`: Seed + direct links (balanced)
-   - `--depth 2+`: Comprehensive but slower and more resource-intensive
-
-5. **Monitor Robot Rules**: Keep `robots.txt` respect enabled to honor website crawling guidelines
-
-6. **Check Logs**: Review output to ensure scanning is working as expected
-
-7. **Test First**: Test on staging environments before scanning production sites
+Robots.txt rules are respected by default. Disabling this with `--ignore-robots` should only be done with explicit permission from the site owner.
 
 ## Output
 
-The Web source emits chunks containing:
+Each crawled page produces a chunk with the following metadata:
 
-- **Page Content**: The raw HTML/text content of each page
-- **Page Title**: Extracted from the `<title>` tag
-- **URL**: The full URL of the crawled page
-- **Depth**: How many links deep the page is from the seed URL
-- **Content-Type**: The MIME type of the content (e.g., `text/html`)
-- **Timestamp**: When the page was crawled (UTC, RFC3339 format)
-
-### Example Metadata
+| Field | Description |
+|-------|-------------|
+| `url` | Full URL of the crawled page |
+| `page_title` | Text content of the `<title>` element |
+| `depth` | Number of hops from the seed URL |
+| `content_type` | MIME type of the response |
+| `timestamp` | Crawl time in UTC RFC3339 format |
 
+Example:
 ```json
 {
   "url": "https://example.com/about",
@@ -183,60 +87,5 @@ The Web source emits chunks containing:
 
 ## Limitations
 
-- **Link Depth Only**: Maximum crawl depth is limited by the `--depth` flag
-  - Deeper scans take longer and consume more memory
-  - Very deep crawls (5+) on large websites may timeout or consume excessive resources
-
-- **Single Domain**: Only crawls the target domain and its subdomains
-  - External links are skipped by design
-  - Run multiple scans for different domains
-
-- **30-Second Timeout**: Hard limit on individual crawl operations
-  - Adjust in code if needed: `context.WithTimeout(context.Background(), 30*time.Second)`
-
-- **No JavaScript Rendering**: Static HTML content only
-  - Websites with JavaScript-rendered content may appear incomplete
-  - Future enhancement: Add JavaScript rendering support
-
-- **No Authentication**: Cannot scan behind login pages
-  - Workaround: Manually extract session cookies and pass them as headers
-  - Future enhancement: Add authentication support
-
-## Troubleshooting
-
-### No Pages Crawled
-
-Check for:
-1. **Invalid URL**: Ensure the URL is valid and accessible
-2. **Network Issues**: Verify internet connectivity
-3. **Robots.txt Block**: The website's `robots.txt` may block all crawling
-4. **No Discoverable Links**: The page may have no links for the crawler to follow
-
-### Slow Crawling
-
-- Increase `concurrency` (but be respectful)
-- Reduce `delay` if appropriate
-- Check your internet connection
-
-## Security Considerations
-
-- **Sensitive Data**: Be cautious when scanning internal or staging environments
-- **Legal Compliance**: Ensure you have authorization before scanning websites
-- **Network Traffic**: Crawling generates significant network traffic and server logs
-
-## Future Enhancements
-
-- [ ] JavaScript rendering support (Puppeteer/Playwright integration)
-- [ ] Authentication support (Basic auth, cookies, form login)
-- [ ] Custom header configuration
-- [ ] Form submission and POST request handling
-- [ ] Incremental crawling with state persistence
-- [ ] Configurable timeout per scan
-- [ ] Rate limiting by content size
-- [ ] Proxy support for scanning through corporate networks
-
-## References
-
-- [Colly Web Scraping Framework](http://go-colly.org/)
-- [Robots.txt Specification](https://en.wikipedia.org/wiki/Robots.txt)
-- [Responsible Web Crawling Guidelines](https://www.robotstxt.org/)
+- Only the target domain and its subdomains are crawled; external links are skipped by design.
+- No support for authenticated pages (login-gated content).

From 717c2eabe944e8cdd49715f7739e473926286246 Mon Sep 17 00:00:00 2001
From: Kashif Khan <kashif.khan@trufflesec.com>
Date: Mon, 30 Mar 2026 16:50:35 +0500
Subject: [PATCH 11/20] Added missed config in engine and rewrite timeout
 comment

---
 main.go                |  2 +-
 pkg/engine/web.go      | 11 +++++++----
 pkg/sources/sources.go |  5 ++---
 3 files changed, 10 insertions(+), 8 deletions(-)

diff --git a/main.go b/main.go
index 106501e87fc8..a7f1e33105eb 100644
--- a/main.go
+++ b/main.go
@@ -281,7 +281,7 @@ var (
 	webCrawl        = webScan.Flag("crawl", "Follow links found on each page. Without this flag only the seed URL(s) are scanned.").Default("false").Bool()
 	webDepth        = webScan.Flag("depth", "Maximum link depth to follow when --crawl is enabled. 1 = seed; 2 = one level deeper; 0 = unlimited.").Default("1").Int()
 	webDelay        = webScan.Flag("delay", "Seconds to wait between requests to the same domain. Increase this to reduce load on the target server.").Default("1").Int()
-	webTimeout      = webScan.Flag("timeout", "Seconds to spend crawling each URL before aborting. Applied per URL when multiple --url flags are given.").Default("30").Int()
+	webTimeout      = webScan.Flag("timeout", "Seconds to spend crawling URLs before aborting. Total time shared across all URLs when multiple --url flags are given.").Default("30").Int()
 	webUserAgent    = webScan.Flag("user-agent", "User-Agent header to send with each request. Defaults to a TruffleHog identifier if not set.").String()
 	webIgnoreRobots = webScan.Flag("ignore-robots", "Ignore robots.txt restrictions. Only use this if you have explicit permission to crawl the target site.").Default("false").Bool()
 
diff --git a/pkg/engine/web.go b/pkg/engine/web.go
index 034fccdb5d0f..310633e45586 100644
--- a/pkg/engine/web.go
+++ b/pkg/engine/web.go
@@ -15,10 +15,13 @@ import (
 // ScanWeb scans a given web connection.
 func (e *Engine) ScanWeb(ctx context.Context, c sources.WebConfig) (sources.JobProgressRef, error) {
 	connection := &sourcespb.Web{
-		Urls:  c.URLs,
-		Crawl: c.Crawl,
-		Depth: int64(c.Depth),
-		Delay: int64(c.Delay),
+		Urls:         c.URLs,
+		Crawl:        c.Crawl,
+		Depth:        int64(c.Depth),
+		Delay:        int64(c.Delay),
+		Timeout:      int64(c.Timeout),
+		UserAgent:    c.UserAgent,
+		IgnoreRobots: c.IgnoreRobots,
 	}
 
 	var conn anypb.Any
diff --git a/pkg/sources/sources.go b/pkg/sources/sources.go
index b5ba5f03099c..b2c84c0e3c55 100644
--- a/pkg/sources/sources.go
+++ b/pkg/sources/sources.go
@@ -500,9 +500,8 @@ type WebConfig struct {
 	// domain. Increase this to reduce load on the target server.
 	Delay int
 
-	// Timeout is the maximum number of seconds to spend crawling a single
-	// seed URL before aborting. Applied independently per URL.
-	// Defaults to 30 seconds if unset or zero.
+	// Timeout is the maximum number of seconds to spend crawling
+	// seeded URLs before aborting. Defaults to 30 seconds if unset or zero.
 	Timeout int
 
 	// UserAgent is the User-Agent header sent with each request.

From 3770cf55f7d91ea0aa2e2245b6ed6a122980cbd0 Mon Sep 17 00:00:00 2001
From: Kashif Khan <kashif.khan@trufflesec.com>
Date: Mon, 30 Mar 2026 16:52:56 +0500
Subject: [PATCH 12/20] fixed lint issues

---
 pkg/sources/web/web.go | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/pkg/sources/web/web.go b/pkg/sources/web/web.go
index 87937b1d6088..01748c97f6b4 100644
--- a/pkg/sources/web/web.go
+++ b/pkg/sources/web/web.go
@@ -107,7 +107,6 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ .
 	eg, _ := errgroup.WithContext(crawlCtx)
 
 	for _, u := range s.conn.GetUrls() {
-		u := u // capture
 		ctx.Logger().V(5).Info("Processing Url", "url", u)
 		webUrlsScanned.WithLabelValues(s.name, jobIDStr).Inc()
 		eg.Go(func() error {
@@ -153,11 +152,13 @@ func (s *Source) crawlURL(ctx context.Context, seedURL string, chunksChan chan *
 	// Users can enable this only when they have explicit permission to crawl the target site.
 	collector.IgnoreRobotsTxt = s.conn.GetIgnoreRobots()
 
-	collector.Limit(&colly.LimitRule{
+	if err := collector.Limit(&colly.LimitRule{
 		DomainGlob:  "*",
 		Parallelism: s.concurrency,
 		Delay:       time.Duration(s.conn.GetDelay()) * time.Second,
-	})
+	}); err != nil {
+		return fmt.Errorf("failed to limit rules to the colly collector: %w", err)
+	}
 
 	// Set up callbacks
 	collector.OnResponse(func(r *colly.Response) {

From 721505f65dc4ce379c46b067fc3427619dccbaea Mon Sep 17 00:00:00 2001
From: Kashif Khan <kashif.khan@trufflesec.com>
Date: Mon, 30 Mar 2026 17:16:56 +0500
Subject: [PATCH 13/20] fixed allowed domains validation

---
 pkg/sources/web/web.go | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/pkg/sources/web/web.go b/pkg/sources/web/web.go
index 01748c97f6b4..92c2291c65f0 100644
--- a/pkg/sources/web/web.go
+++ b/pkg/sources/web/web.go
@@ -101,7 +101,7 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ .
 	jobIDStr := fmt.Sprint(s.jobId)
 
 	// Create a background context for crawling (independent of incoming ctx).
-	crawlCtx, cancel := context.WithTimeout(context.Background(), time.Duration(s.conn.GetTimeout())*time.Second)
+	crawlCtx, cancel := context.WithTimeout(ctx, time.Duration(s.conn.GetTimeout())*time.Second)
 	defer cancel()
 
 	eg, _ := errgroup.WithContext(crawlCtx)
@@ -139,7 +139,6 @@ func (s *Source) crawlURL(ctx context.Context, seedURL string, chunksChan chan *
 	// docs: http://go-colly.org/docs/introduction/configuration/
 	collector := colly.NewCollector(
 		colly.UserAgent(s.conn.GetUserAgent()),
-		colly.AllowedDomains(parsedURL.Hostname(), fmt.Sprintf("*.%s", parsedURL.Hostname())), // with subdomains
 		colly.Async(true),
 	)
 
@@ -160,6 +159,15 @@ func (s *Source) crawlURL(ctx context.Context, seedURL string, chunksChan chan *
 		return fmt.Errorf("failed to limit rules to the colly collector: %w", err)
 	}
 
+	// request validations
+	collector.OnRequest(func(r *colly.Request) {
+		host := r.URL.Hostname()
+		if host != parsedURL.Hostname() && !strings.HasSuffix(host, parsedURL.Hostname()) {
+			ctx.Logger().V(5).Info("blocked by domain filter", "url", r.URL.String())
+			r.Abort()
+		}
+	})
+
 	// Set up callbacks
 	collector.OnResponse(func(r *colly.Response) {
 		ctx.Logger().Info("Response received")

From 5c84a6bf2ea094f292b201d1d2bae2da9b7e50a4 Mon Sep 17 00:00:00 2001
From: Kashif Khan <kashif.khan@trufflesec.com>
Date: Mon, 30 Mar 2026 17:18:12 +0500
Subject: [PATCH 14/20] fixed comment

---
 pkg/sources/web/web.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pkg/sources/web/web.go b/pkg/sources/web/web.go
index 92c2291c65f0..d37d5e7e7eb6 100644
--- a/pkg/sources/web/web.go
+++ b/pkg/sources/web/web.go
@@ -100,7 +100,7 @@ func (s *Source) Init(ctx context.Context, name string, jobId sources.JobID, sou
 func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ ...sources.ChunkingTarget) error {
 	jobIDStr := fmt.Sprint(s.jobId)
 
-	// Create a background context for crawling (independent of incoming ctx).
+	// Create a new context with timeout.
 	crawlCtx, cancel := context.WithTimeout(ctx, time.Duration(s.conn.GetTimeout())*time.Second)
 	defer cancel()
 

From aade78b9cc18434ae4a4821f7878603a94d56f92 Mon Sep 17 00:00:00 2001
From: Kashif Khan <kashif.khan@trufflesec.com>
Date: Mon, 30 Mar 2026 17:24:22 +0500
Subject: [PATCH 15/20] fixed sub-domain filter

---
 pkg/sources/web/web.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pkg/sources/web/web.go b/pkg/sources/web/web.go
index d37d5e7e7eb6..304ae90b4f6c 100644
--- a/pkg/sources/web/web.go
+++ b/pkg/sources/web/web.go
@@ -162,7 +162,7 @@ func (s *Source) crawlURL(ctx context.Context, seedURL string, chunksChan chan *
 	// request validations
 	collector.OnRequest(func(r *colly.Request) {
 		host := r.URL.Hostname()
-		if host != parsedURL.Hostname() && !strings.HasSuffix(host, parsedURL.Hostname()) {
+		if host != parsedURL.Hostname() && !strings.HasSuffix(host, "."+parsedURL.Hostname()) {
 			ctx.Logger().V(5).Info("blocked by domain filter", "url", r.URL.String())
 			r.Abort()
 		}

From 725b65eff5f991534f1f9a4b6549d878b254a763 Mon Sep 17 00:00:00 2001
From: Kashif Khan <kashif.khan@trufflesec.com>
Date: Thu, 9 Apr 2026 13:10:44 +0500
Subject: [PATCH 16/20] fixed comments

---
 pkg/sources/web/web.go      | 21 +++++++++++++-
 pkg/sources/web/web_test.go | 56 +++++++++++++++++++++++++++++++++++++
 2 files changed, 76 insertions(+), 1 deletion(-)

diff --git a/pkg/sources/web/web.go b/pkg/sources/web/web.go
index 304ae90b4f6c..db00603427e0 100644
--- a/pkg/sources/web/web.go
+++ b/pkg/sources/web/web.go
@@ -84,9 +84,19 @@ func (s *Source) Init(ctx context.Context, name string, jobId sources.JobID, sou
 
 	// Validate URLs format
 	for _, u := range s.conn.GetUrls() {
-		if _, err := url.Parse(u); err != nil {
+		parsedURL, err := url.Parse(u)
+		if err != nil {
 			return fmt.Errorf("invalid URL %q: %w", u, err)
 		}
+		if parsedURL.Scheme == "" {
+			return fmt.Errorf("invalid URL %q: missing scheme (must be http or https)", u)
+		}
+		if parsedURL.Scheme != "http" && parsedURL.Scheme != "https" {
+			return fmt.Errorf("invalid URL %q: unsupported scheme %q (must be http or https)", u, parsedURL.Scheme)
+		}
+		if parsedURL.Host == "" {
+			return fmt.Errorf("invalid URL %q: missing host", u)
+		}
 	}
 
 	// metrics
@@ -105,6 +115,7 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ .
 	defer cancel()
 
 	eg, _ := errgroup.WithContext(crawlCtx)
+	eg.SetLimit(s.concurrency)
 
 	for _, u := range s.conn.GetUrls() {
 		ctx.Logger().V(5).Info("Processing Url", "url", u)
@@ -159,6 +170,14 @@ func (s *Source) crawlURL(ctx context.Context, seedURL string, chunksChan chan *
 		return fmt.Errorf("failed to limit rules to the colly collector: %w", err)
 	}
 
+	// Wire the collector to our cancellable context so in-flight HTTP requests
+	// are cancelled immediately when the context deadline fires.
+	collector.Context = ctx
+
+	// Per-request HTTP timeout as a safety net for individual slow responses.
+	// This is separate from the overall crawl timeout controlled by ctx.
+	collector.SetRequestTimeout(30 * time.Second)
+
 	// request validations
 	collector.OnRequest(func(r *colly.Request) {
 		host := r.URL.Hostname()
diff --git a/pkg/sources/web/web_test.go b/pkg/sources/web/web_test.go
index 07ae70be2468..3b38df81053d 100644
--- a/pkg/sources/web/web_test.go
+++ b/pkg/sources/web/web_test.go
@@ -88,6 +88,61 @@ func TestInit_ZeroConcurrencyDefaultsToOne(t *testing.T) {
 	assert.Equal(t, 1, s.concurrency)
 }
 
+// URL validation
+
+func TestInit_InvalidURL_MissingScheme(t *testing.T) {
+	conn := &anypb.Any{}
+	require.NoError(t, conn.MarshalFrom(&sourcespb.Web{
+		Urls: []string{"example.com"},
+	}))
+	s := &Source{}
+	err := s.Init(context.TODO(), "test", 0, 0, false, conn, 1)
+	assert.Error(t, err)
+	assert.Contains(t, err.Error(), "missing scheme")
+}
+
+func TestInit_InvalidURL_UnsupportedScheme(t *testing.T) {
+	conn := &anypb.Any{}
+	require.NoError(t, conn.MarshalFrom(&sourcespb.Web{
+		Urls: []string{"ftp://example.com"},
+	}))
+	s := &Source{}
+	err := s.Init(context.TODO(), "test", 0, 0, false, conn, 1)
+	assert.Error(t, err)
+	assert.Contains(t, err.Error(), "unsupported scheme")
+}
+
+func TestInit_InvalidURL_MissingHost(t *testing.T) {
+	conn := &anypb.Any{}
+	require.NoError(t, conn.MarshalFrom(&sourcespb.Web{
+		Urls: []string{"http://"},
+	}))
+	s := &Source{}
+	err := s.Init(context.TODO(), "test", 0, 0, false, conn, 1)
+	assert.Error(t, err)
+	assert.Contains(t, err.Error(), "missing host")
+}
+
+func TestInit_ValidURL_HTTP(t *testing.T) {
+	conn := &anypb.Any{}
+	require.NoError(t, conn.MarshalFrom(&sourcespb.Web{
+		Urls: []string{"http://example.com"},
+	}))
+	s := &Source{}
+	err := s.Init(context.TODO(), "test", 0, 0, false, conn, 1)
+	assert.NoError(t, err)
+}
+
+func TestInit_ValidURL_HTTPS(t *testing.T) {
+	conn := &anypb.Any{}
+	require.NoError(t, conn.MarshalFrom(&sourcespb.Web{
+		Urls: []string{"https://example.com"},
+	}))
+	s := &Source{}
+	err := s.Init(context.TODO(), "test", 0, 0, false, conn, 1)
+	assert.NoError(t, err)
+}
+
 // Happy path
 
 func TestChunks_HappyPath(t *testing.T) {
@@ -429,3 +484,4 @@ func TestChunks_IgnoreRobotsTxt(t *testing.T) {
 
 	assert.True(t, secretVisited, "/secret should be crawled when IgnoreRobots=true")
 }
+

From 40e8d6bce20bcf5272c258876af2c692b92a887f Mon Sep 17 00:00:00 2001
From: Kashif Khan <kashif.khan@trufflesec.com>
Date: Thu, 9 Apr 2026 13:31:59 +0500
Subject: [PATCH 17/20] fixed comment

---
 pkg/sources/web/web.go      | 13 +++++++++----
 pkg/sources/web/web_test.go | 20 ++++++++++++++++++++
 2 files changed, 29 insertions(+), 4 deletions(-)

diff --git a/pkg/sources/web/web.go b/pkg/sources/web/web.go
index db00603427e0..c56ca7abab43 100644
--- a/pkg/sources/web/web.go
+++ b/pkg/sources/web/web.go
@@ -228,20 +228,25 @@ func (s *Source) crawlURL(ctx context.Context, seedURL string, chunksChan chan *
 		})
 	}
 
-	// Create a channel to signal when the crawl is done.
-	done := make(chan struct{})
+	// Create a channel to propagate errors from the crawl goroutine.
+	done := make(chan error)
 	go func() {
 		ctx.Logger().Info("Starting crawl")
 		if err := collector.Visit(seedURL); err != nil {
 			ctx.Logger().Error(err, "Visit failed")
+			done <- err
+			return
 		}
 		collector.Wait() // blocks until all requests finish
-		close(done)
+		done <- nil      // Signal successful completion
 	}()
 
 	// Wait for either crawl to finish or context cancellation.
 	select {
-	case <-done:
+	case err := <-done:
+		if err != nil {
+			return err
+		}
 		ctx.Logger().Info("Crawl finished normally")
 		return nil
 	case <-ctx.Done():
diff --git a/pkg/sources/web/web_test.go b/pkg/sources/web/web_test.go
index 3b38df81053d..90d393d6650d 100644
--- a/pkg/sources/web/web_test.go
+++ b/pkg/sources/web/web_test.go
@@ -143,6 +143,26 @@ func TestInit_ValidURL_HTTPS(t *testing.T) {
 	assert.NoError(t, err)
 }
 
+// Visit error propagation
+
+func TestChunks_VisitErrorPropagated(t *testing.T) {
+	// Test that Visit() errors on the seed URL are propagated, not silently swallowed.
+	// Point to a URL that will definitely fail to connect.
+	failURL := "http://localhost:1"  // Port 1 is unlikely to have a service; connection should be refused
+
+	s := initSource(t, &sourcespb.Web{Urls: []string{failURL}, Timeout: 2}, 1)
+	chunksChan := make(chan *sources.Chunk, 16)
+
+	// Chunks() should return an error, not nil.
+	err := s.Chunks(context.TODO(), chunksChan)
+	close(chunksChan)
+
+	// The error should be non-nil because the seed URL is unreachable.
+	assert.Error(t, err, "expected Visit() error to be propagated")
+	// We don't check the exact error message since it can vary by OS/network,
+	// but it should be a network error, not a context error.
+}
+
 // Happy path
 
 func TestChunks_HappyPath(t *testing.T) {

From 28c2a1c5c03df82184e1d175c173555e03f3a1e8 Mon Sep 17 00:00:00 2001
From: Kashif Khan <kashif.khan@trufflesec.com>
Date: Thu, 9 Apr 2026 13:44:12 +0500
Subject: [PATCH 18/20] fixed metric count

---
 pkg/sources/web/web.go | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/pkg/sources/web/web.go b/pkg/sources/web/web.go
index c56ca7abab43..cd0a4da8e9c4 100644
--- a/pkg/sources/web/web.go
+++ b/pkg/sources/web/web.go
@@ -108,8 +108,6 @@ func (s *Source) Init(ctx context.Context, name string, jobId sources.JobID, sou
 
 // Chunks emits data over a channel that is decoded and scanned for secrets.
 func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ ...sources.ChunkingTarget) error {
-	jobIDStr := fmt.Sprint(s.jobId)
-
 	// Create a new context with timeout.
 	crawlCtx, cancel := context.WithTimeout(ctx, time.Duration(s.conn.GetTimeout())*time.Second)
 	defer cancel()
@@ -119,7 +117,6 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ .
 
 	for _, u := range s.conn.GetUrls() {
 		ctx.Logger().V(5).Info("Processing Url", "url", u)
-		webUrlsScanned.WithLabelValues(s.name, jobIDStr).Inc()
 		eg.Go(func() error {
 			return s.crawlURL(crawlCtx, u, chunksChan)
 		})
@@ -190,6 +187,9 @@ func (s *Source) crawlURL(ctx context.Context, seedURL string, chunksChan chan *
 	// Set up callbacks
 	collector.OnResponse(func(r *colly.Response) {
 		ctx.Logger().Info("Response received")
+		// Increment metric for each page actually fetched.
+		jobIDStr := fmt.Sprint(s.jobId)
+		webUrlsScanned.WithLabelValues(s.name, jobIDStr).Inc()
 		if err := s.processChunk(ctx, r, chunksChan); err != nil {
 			ctx.Logger().Error(err, "error processing page")
 		}

From 4d7fa57473c864668b55ca5892191c4167008064 Mon Sep 17 00:00:00 2001
From: Kashif Khan <kashif.khan@trufflesec.com>
Date: Thu, 9 Apr 2026 15:38:55 +0500
Subject: [PATCH 19/20] web: Fix timeouts, validation, concurrency, error
 handling, and observability

---
 pkg/sources/web/web.go      | 24 ++++++++++++----
 pkg/sources/web/web_test.go | 55 +++++++++++++++++++++++++++++++++++++
 2 files changed, 74 insertions(+), 5 deletions(-)

diff --git a/pkg/sources/web/web.go b/pkg/sources/web/web.go
index cd0a4da8e9c4..45903a09ff46 100644
--- a/pkg/sources/web/web.go
+++ b/pkg/sources/web/web.go
@@ -82,8 +82,14 @@ func (s *Source) Init(ctx context.Context, name string, jobId sources.JobID, sou
 		return errors.New("no URL provided")
 	}
 
-	// Validate URLs format
+	// Validate URLs format and detect duplicates
+	seen := make(map[string]struct{}, len(s.conn.GetUrls()))
 	for _, u := range s.conn.GetUrls() {
+		if _, dup := seen[u]; dup {
+			return fmt.Errorf("duplicate URL %q", u)
+		}
+		seen[u] = struct{}{}
+
 		parsedURL, err := url.Parse(u)
 		if err != nil {
 			return fmt.Errorf("invalid URL %q: %w", u, err)
@@ -112,18 +118,25 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ .
 	crawlCtx, cancel := context.WithTimeout(ctx, time.Duration(s.conn.GetTimeout())*time.Second)
 	defer cancel()
 
-	eg, _ := errgroup.WithContext(crawlCtx)
+	var eg errgroup.Group
 	eg.SetLimit(s.concurrency)
+	scanErrs := sources.NewScanErrors()
 
 	for _, u := range s.conn.GetUrls() {
 		ctx.Logger().V(5).Info("Processing Url", "url", u)
+		u := u // capture loop variable
 		eg.Go(func() error {
-			return s.crawlURL(crawlCtx, u, chunksChan)
+			if err := s.crawlURL(crawlCtx, u, chunksChan); err != nil {
+				ctx.Logger().Error(err, "Crawl failed", "url", u)
+				scanErrs.Add(fmt.Errorf("%s: %w", u, err))
+			}
+			return nil // continue on error; don't propagate to errgroup
 		})
 	}
 
-	if err := eg.Wait(); err != nil {
-		ctx.Logger().Error(err, "One or more crawls failed")
+	eg.Wait()
+
+	if err := scanErrs.Errors(); err != nil {
 		return err
 	}
 
@@ -148,6 +161,7 @@ func (s *Source) crawlURL(ctx context.Context, seedURL string, chunksChan chan *
 	collector := colly.NewCollector(
 		colly.UserAgent(s.conn.GetUserAgent()),
 		colly.Async(true),
+		colly.MaxBodySize(10 * 1024 * 1024), // 10 MB limit per response
 	)
 
 	// Apply depth limit only when crawling is enabled and a positive depth is set.
diff --git a/pkg/sources/web/web_test.go b/pkg/sources/web/web_test.go
index 90d393d6650d..0028a2a12823 100644
--- a/pkg/sources/web/web_test.go
+++ b/pkg/sources/web/web_test.go
@@ -143,6 +143,61 @@ func TestInit_ValidURL_HTTPS(t *testing.T) {
 	assert.NoError(t, err)
 }
 
+func TestInit_DuplicateURL(t *testing.T) {
+	conn := &anypb.Any{}
+	require.NoError(t, conn.MarshalFrom(&sourcespb.Web{
+		Urls: []string{"http://example.com", "http://example.com"},
+	}))
+	s := &Source{}
+	err := s.Init(context.TODO(), "test", 0, 0, false, conn, 1)
+	assert.Error(t, err)
+	assert.Contains(t, err.Error(), "duplicate URL")
+}
+
+// Partial failure and error handling
+
+func TestChunks_PartialFailure(t *testing.T) {
+	// Test that when one URL fails and another succeeds, we still process the successful one
+	// and return an error at the end.
+
+	goodSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "text/html")
+		fmt.Fprint(w, `<html><head><title>Good</title></head><body>Good data</body></html>`)
+	}))
+	defer goodSrv.Close()
+
+	// This URL will fail (port 1 is unreachable)
+	badURL := "http://localhost:1"
+
+	s := initSource(t, &sourcespb.Web{
+		Urls: []string{badURL, goodSrv.URL},
+		Timeout: 2,
+	}, 1)
+
+	chunksChan := make(chan *sources.Chunk, 16)
+
+	// Chunks should return an error due to the bad URL
+	err := s.Chunks(context.TODO(), chunksChan)
+	close(chunksChan)
+
+	// Collect chunks that were successfully emitted
+	var chunks []*sources.Chunk
+	for c := range chunksChan {
+		chunks = append(chunks, c)
+	}
+
+	// Should have an error (the bad URL)
+	assert.Error(t, err, "expected error from unreachable URL")
+
+	// But should still have processed the good URL
+	assert.NotEmpty(t, chunks, "expected chunks from the reachable URL despite one failure")
+
+	// Verify the chunk is from the good URL
+	meta := chunks[0].SourceMetadata.Data.(*source_metadatapb.MetaData_Web)
+	assert.Contains(t, meta.Web.Url, "127.0.0.1", "chunk should be from the working URL")
+	assert.Equal(t, "Good", meta.Web.PageTitle)
+}
+
 // Visit error propagation
 
 func TestChunks_VisitErrorPropagated(t *testing.T) {

From ada4aee0269138737fdfd1ab8c755757cc4fcfec Mon Sep 17 00:00:00 2001
From: Kashif Khan <kashif.khan@trufflesec.com>
Date: Thu, 9 Apr 2026 15:42:04 +0500
Subject: [PATCH 20/20] fixed linter

---
 pkg/sources/web/web.go | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/pkg/sources/web/web.go b/pkg/sources/web/web.go
index 45903a09ff46..4243aadd5c6f 100644
--- a/pkg/sources/web/web.go
+++ b/pkg/sources/web/web.go
@@ -124,7 +124,6 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ .
 
 	for _, u := range s.conn.GetUrls() {
 		ctx.Logger().V(5).Info("Processing Url", "url", u)
-		u := u // capture loop variable
 		eg.Go(func() error {
 			if err := s.crawlURL(crawlCtx, u, chunksChan); err != nil {
 				ctx.Logger().Error(err, "Crawl failed", "url", u)
@@ -134,7 +133,7 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ .
 		})
 	}
 
-	eg.Wait()
+	_ = eg.Wait()
 
 	if err := scanErrs.Errors(); err != nil {
 		return err
@@ -161,7 +160,7 @@ func (s *Source) crawlURL(ctx context.Context, seedURL string, chunksChan chan *
 	collector := colly.NewCollector(
 		colly.UserAgent(s.conn.GetUserAgent()),
 		colly.Async(true),
-		colly.MaxBodySize(10 * 1024 * 1024), // 10 MB limit per response
+		colly.MaxBodySize(10*1024*1024), // 10 MB limit per response
 	)
 
 	// Apply depth limit only when crawling is enabled and a positive depth is set.