diff --git a/pkg/detectors/bitbucketdatacenter/bitbucketdatacenter.go b/pkg/detectors/bitbucketdatacenter/bitbucketdatacenter.go new file mode 100644 index 000000000000..6e3ef0d49d1d --- /dev/null +++ b/pkg/detectors/bitbucketdatacenter/bitbucketdatacenter.go @@ -0,0 +1,140 @@ +package bitbucketdatacenter + +import ( + "context" + "fmt" + "io" + "net/http" + "strings" + + regexp "github.com/wasilibs/go-re2" + + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" +) + +type Scanner struct { + client *http.Client + detectors.DefaultMultiPartCredentialProvider + detectors.EndpointSetter +} + +// Ensure the Scanner satisfies the interface at compile time. +var _ detectors.Detector = (*Scanner)(nil) +var _ detectors.EndpointCustomizer = (*Scanner)(nil) + +var ( + defaultClient = detectors.DetectorHttpClientWithNoLocalAddresses + + // Bitbucket pat start with BBDC- prefix + // and are usually between the length of 40-50 character + // consisting of both alphanumeric and some special character like +, _, @ and etc + userPat = regexp.MustCompile(`\b(BBDC-[A-Za-z0-9+/@_-]{40,50})(?:[^A-Za-z0-9+/@_-]|$)`) + + urlPat = regexp.MustCompile(detectors.PrefixRegex([]string{"atlassian", "bitbucket"}) + `(https://[a-zA-Z0-9.-]+(?::\d+)?)`) +) + +func (s Scanner) Keywords() []string { + return []string{"BBDC-"} +} + +// FromData will find and optionally verify HashiCorp Vault AppRole secrets in a given set of bytes. +func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) { + dataStr := string(data) + + var uniqueSecretPat = make(map[string]struct{}) + for _, match := range userPat.FindAllStringSubmatch(dataStr, -1) { + secretPat := strings.TrimSpace(match[1]) + uniqueSecretPat[secretPat] = struct{}{} + } + + if len(uniqueSecretPat) == 0 { + return results, nil + } + + foundURLs := make(map[string]struct{}) + for _, match := range urlPat.FindAllStringSubmatch(dataStr, -1) { + foundURLs[match[1]] = struct{}{} + } + + endpoints := make([]string, 0, len(foundURLs)) + for endpoint := range foundURLs { + endpoints = append(endpoints, endpoint) + } + + var uniqueUrls = make(map[string]struct{}) + for _, endpoint := range s.Endpoints(endpoints...) { + uniqueUrls[endpoint] = struct{}{} + } + + // create combination results that can be verified + for secret := range uniqueSecretPat { + for bitBucketURL := range uniqueUrls { + s1 := detectors.Result{ + DetectorType: detector_typepb.DetectorType_BitbucketDataCenter, + Raw: []byte(secret), + RawV2: []byte(fmt.Sprintf("%s:%s", secret, bitBucketURL)), + ExtraData: map[string]string{ + "URL": bitBucketURL, + }, + } + + if verify { + client := s.client + if client == nil { + client = defaultClient + } + + isVerified, verificationErr := verifyMatch(ctx, client, secret, bitBucketURL) + s1.Verified = isVerified + s1.SetVerificationError(verificationErr, secret, bitBucketURL) + } + results = append(results, s1) + } + } + return results, nil +} + +func verifyMatch(ctx context.Context, client *http.Client, secretPat, baseURL string) (bool, error) { + u, err := detectors.ParseURLAndStripPathAndParams(baseURL) + if err != nil { + return false, err + } + u.Path = "rest/api/1.0/projects" + q := u.Query() + q.Set("limit", "1") + u.RawQuery = q.Encode() + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), http.NoBody) + if err != nil { + return false, err + } + req.Header.Set("Accept", "application/json") + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", secretPat)) + resp, err := client.Do(req) + if err != nil { + return false, err + } + + defer func() { + _, _ = io.Copy(io.Discard, resp.Body) + _ = resp.Body.Close() + }() + + switch resp.StatusCode { + case http.StatusOK: + return true, nil + case http.StatusUnauthorized: + return false, nil + default: + return false, fmt.Errorf("unexpected HTTP response status %d", resp.StatusCode) + } +} + +func (s Scanner) Type() detector_typepb.DetectorType { + return detector_typepb.DetectorType_BitbucketDataCenter +} + +func (s Scanner) Description() string { + return "Bitbucket is a Git repository hosting service by Atlassian. Bitbucket PATs are used to authenticate bitbucket data center(on prem) rest endpoint requests." +} diff --git a/pkg/detectors/bitbucketdatacenter/bitbucketdatacenter_test.go b/pkg/detectors/bitbucketdatacenter/bitbucketdatacenter_test.go new file mode 100644 index 000000000000..51f605659cfb --- /dev/null +++ b/pkg/detectors/bitbucketdatacenter/bitbucketdatacenter_test.go @@ -0,0 +1,386 @@ +package bitbucketdatacenter + +import ( + "context" + "fmt" + "net/http" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gopkg.in/h2non/gock.v1" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" +) + +func TestBitbucketDataCenter_Pattern(t *testing.T) { + d := Scanner{} + d.UseCloudEndpoint(true) + d.UseFoundEndpoints(true) + + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) + + tests := []struct { + name string + input string + want []string + }{ + { + name: "invalid - pat only no url", + input: ` + BBDC-MTM5NDkzNDI3MTgzOrTObCIIEXN0tpQYAc4bhG+RUqwz + `, + want: nil, + }, + { + name: "invalid - pat and unrelated url", + input: ` + BBDC-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA + https://example.com/api + `, + want: nil, + }, + { + name: "valid - single pat single url", + input: ` + atlassian bitbucket running at https://git.company.com:7990/projects/PROJ/repos/app + BBDC-MTk4MDE0MzAyMDIzOvvP+lDf5edYvDgggvyzpmiXkF0A + `, + want: []string{ + "BBDC-MTk4MDE0MzAyMDIzOvvP+lDf5edYvDgggvyzpmiXkF0A:https://git.company.com:7990", + }, + }, + { + name: "valid - multiple pats single url", + input: ` + bitbucket server at https://git.company.com:7990/scm/proj/repo.git + + BBDC-1111111111111111111111111111111111111111 + BBDC-2222222222222222222222222222222222222222 + `, + want: []string{ + "BBDC-1111111111111111111111111111111111111111:https://git.company.com:7990", + "BBDC-2222222222222222222222222222222222222222:https://git.company.com:7990", + }, + }, + { + name: "valid - single pat multiple urls", + input: ` + atlassian bitbucket instance: + BBDC-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA + bitbucket = https://git.company.com:7990/scm/proj/repo.git + bitbucket = https://git.company2.com:7990/scm/proj/repo.git + `, + want: []string{ + "BBDC-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA:https://git.company.com:7990", + "BBDC-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA:https://git.company2.com:7990", + }, + }, + { + name: "invalid - short pat", + input: ` + BBDC-1234 + https://git.company.com:7990 + `, + want: nil, + }, + { + name: "invalid - uppercase pat", + input: ` + BBDC-MTM5NDKZNDI3MTgzORTOBCCIIEXN0TPQYAC4BHG + https://git.company.com:7990 + `, + want: nil, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { + t.Errorf( + "test %q failed: expected keywords %v to be found", + test.name, + d.Keywords(), + ) + return + } + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + require.NoError(t, err) + + actual := make(map[string]struct{}) + for _, r := range results { + actual[string(r.RawV2)] = struct{}{} + } + + expected := make(map[string]struct{}) + for _, v := range test.want { + expected[v] = struct{}{} + } + + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff (-want +got):\n%s", test.name, diff) + } + }) + } +} + +func TestBitbucketDataCenterPAT_FromData(t *testing.T) { + client := common.SaneHttpClient() + + d := Scanner{client: client} + testEndpoint := "https://git.company.com" + testToken := "BBDC-OTE2MTAxMzgwNTgxOs8VegSPPzv+A9lGK3bbnwOFCkhj" + _ = d.SetConfiguredEndpoints(testEndpoint) + d.UseFoundEndpoints(false) + + defer gock.Off() + defer gock.RestoreClient(client) + gock.InterceptClient(client) + + tests := []struct { + name string + setup func() + data string + verify bool + wantResults int + wantVerified bool + wantVerificationErr bool + }{ + { + name: "found, verified with confiured endpoint", + setup: func() { + gock.New(testEndpoint). + Get("/rest/api/1.0/projects"). + MatchHeader("Authorization", fmt.Sprintf("Bearer %s", testToken)). + Reply(http.StatusOK). + JSON(map[string]any{ + "size": 0, + "limit": 1, + "isLastPage": true, + "values": "", + "start": 0, + }) + }, + data: fmt.Sprintf("bitbucket token: %s", testToken), + verify: true, + wantResults: 1, + wantVerified: true, + }, + { + name: "found, unverified (401)", + setup: func() { + gock.New(testEndpoint). + Get("/rest/api/1.0/projects"). + MatchHeader("Authorization", fmt.Sprintf("Bearer %s", testToken)). + Reply(http.StatusUnauthorized) + }, + data: fmt.Sprintf("bitbucket token: %s", testToken), + verify: true, + wantResults: 1, + wantVerified: false, + }, + { + name: "not found", + setup: func() {}, + data: "bitbucket config: nothing here", + verify: true, + wantResults: 0, + }, + { + name: "found, verification error on unexpected status", + setup: func() { + gock.New(testEndpoint). + Get("/rest/api/1.0/projects"). + MatchHeader("Authorization", fmt.Sprintf("Bearer %s", testToken)). + Reply(http.StatusInternalServerError) + }, + data: fmt.Sprintf("bitbucket token: %s", testToken), + verify: true, + wantResults: 1, + wantVerified: false, + wantVerificationErr: true, + }, + { + name: "found, verification error on timeout", + setup: func() { + gock.New(testEndpoint). + Get("/rest/api/1.0/projects"). + MatchHeader("Authorization", fmt.Sprintf("Bearer %s", testToken)). + Reply(http.StatusOK). + Delay(2 * time.Second) + }, + data: fmt.Sprintf("bitbucket token: %s", testToken), + verify: true, + wantResults: 1, + wantVerified: false, + wantVerificationErr: true, + }, + { + name: "found, no verify", + setup: func() {}, + data: fmt.Sprintf("bitbucket token: %s", testToken), + verify: false, + wantResults: 1, + wantVerified: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gock.Flush() + tt.setup() + + ctx := context.Background() + if tt.wantVerificationErr { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, 100*time.Millisecond) + defer cancel() + } + + results, err := d.FromData(ctx, tt.verify, []byte(tt.data)) + require.NoError(t, err) + require.Len(t, results, tt.wantResults) + + for _, result := range results { + assert.Equal(t, detector_typepb.DetectorType_BitbucketDataCenter, result.DetectorType) + assert.NotEmpty(t, result.Raw) + assert.Equal(t, tt.wantVerified, result.Verified) + assert.Equal(t, tt.wantVerificationErr, result.VerificationError() != nil) + } + }) + } +} + +func TestBitbucketDataCenterPAT_FromData_WithoutConfiguredEndpoint(t *testing.T) { + client := common.SaneHttpClient() + + d := Scanner{client: client} + testEndpoint := "https://git.company.com" + testToken := "BBDC-OTE2MTAxMzgwNTgxOs8VegSPPzv+A9lGK3bbnwOFCkhj" + d.UseFoundEndpoints(true) + + defer gock.Off() + defer gock.RestoreClient(client) + gock.InterceptClient(client) + + tests := []struct { + name string + setup func() + data string + verify bool + wantResults int + wantVerified bool + wantVerificationErr bool + }{ + { + name: "found, verified with confiured endpoint", + setup: func() { + gock.New(testEndpoint). + Get("/rest/api/1.0/projects"). + MatchHeader("Authorization", fmt.Sprintf("Bearer %s", testToken)). + Reply(http.StatusOK). + JSON(map[string]any{ + "size": 0, + "limit": 1, + "isLastPage": true, + "values": "", + "start": 0, + }) + }, + data: fmt.Sprintf("bitbucket url %s token: %s", testEndpoint, testToken), + verify: true, + wantResults: 1, + wantVerified: true, + }, + { + name: "found, unverified (401)", + setup: func() { + gock.New(testEndpoint). + Get("/rest/api/1.0/projects"). + MatchHeader("Authorization", fmt.Sprintf("Bearer %s", testToken)). + Reply(http.StatusUnauthorized) + }, + data: fmt.Sprintf("bitbucket url %s token: %s", testEndpoint, testToken), + verify: true, + wantResults: 1, + wantVerified: false, + }, + { + name: "not found", + setup: func() {}, + data: "bitbucket config: nothing here", + verify: true, + wantResults: 0, + }, + { + name: "found, verification error on unexpected status", + setup: func() { + gock.New(testEndpoint). + Get("/rest/api/1.0/projects"). + MatchHeader("Authorization", fmt.Sprintf("Bearer %s", testToken)). + Reply(http.StatusInternalServerError) + }, + data: fmt.Sprintf("bitbucket url %s token: %s", testEndpoint, testToken), + verify: true, + wantResults: 1, + wantVerified: false, + wantVerificationErr: true, + }, + { + name: "found, verification error on timeout", + setup: func() { + gock.New(testEndpoint). + Get("/rest/api/1.0/projects"). + MatchHeader("Authorization", fmt.Sprintf("Bearer %s", testToken)). + Reply(http.StatusOK). + Delay(2 * time.Second) + }, + data: fmt.Sprintf("bitbucket url %s token: %s", testEndpoint, testToken), + verify: true, + wantResults: 1, + wantVerified: false, + wantVerificationErr: true, + }, + { + name: "found, no verify", + setup: func() {}, + data: fmt.Sprintf("bitbucket url %s token: %s", testEndpoint, testToken), + verify: false, + wantResults: 1, + wantVerified: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gock.Flush() + tt.setup() + + ctx := context.Background() + if tt.wantVerificationErr { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, 100*time.Millisecond) + defer cancel() + } + + results, err := d.FromData(ctx, tt.verify, []byte(tt.data)) + require.NoError(t, err) + require.Len(t, results, tt.wantResults) + + for _, result := range results { + assert.Equal(t, detector_typepb.DetectorType_BitbucketDataCenter, result.DetectorType) + assert.NotEmpty(t, result.Raw) + assert.Equal(t, tt.wantVerified, result.Verified) + assert.Equal(t, tt.wantVerificationErr, result.VerificationError() != nil) + } + }) + } +} diff --git a/pkg/pb/detector_typepb/detector_type.pb.go b/pkg/pb/detector_typepb/detector_type.pb.go index 44725851e6aa..56ac938400c7 100644 --- a/pkg/pb/detector_typepb/detector_type.pb.go +++ b/pkg/pb/detector_typepb/detector_type.pb.go @@ -1098,6 +1098,7 @@ const ( DetectorType_ArtifactoryReferenceToken DetectorType = 1042 DetectorType_DatadogApikey DetectorType = 1043 DetectorType_ShopifyOAuth DetectorType = 1044 + DetectorType_BitbucketDataCenter DetectorType = 1045 ) // Enum value maps for DetectorType. @@ -2144,6 +2145,7 @@ var ( 1042: "ArtifactoryReferenceToken", 1043: "DatadogApikey", 1044: "ShopifyOAuth", + 1045: "BitbucketDataCenter", } DetectorType_value = map[string]int32{ "Alibaba": 0, @@ -3187,6 +3189,7 @@ var ( "ArtifactoryReferenceToken": 1042, "DatadogApikey": 1043, "ShopifyOAuth": 1044, + "BitbucketDataCenter": 1045, } ) @@ -3222,7 +3225,7 @@ var File_detector_type_proto protoreflect.FileDescriptor var file_detector_type_proto_rawDesc = []byte{ 0x0a, 0x13, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0d, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x5f, - 0x74, 0x79, 0x70, 0x65, 0x2a, 0xbf, 0x87, 0x01, 0x0a, 0x0c, 0x44, 0x65, 0x74, 0x65, 0x63, 0x74, + 0x74, 0x79, 0x70, 0x65, 0x2a, 0xd9, 0x87, 0x01, 0x0a, 0x0c, 0x44, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x41, 0x6c, 0x69, 0x62, 0x61, 0x62, 0x61, 0x10, 0x00, 0x12, 0x08, 0x0a, 0x04, 0x41, 0x4d, 0x51, 0x50, 0x10, 0x01, 0x12, 0x07, 0x0a, 0x03, 0x41, 0x57, 0x53, 0x10, 0x02, 0x12, 0x09, 0x0a, 0x05, 0x41, 0x7a, 0x75, 0x72, 0x65, 0x10, @@ -4306,12 +4309,13 @@ var file_detector_type_proto_rawDesc = []byte{ 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x10, 0x92, 0x08, 0x12, 0x12, 0x0a, 0x0d, 0x44, 0x61, 0x74, 0x61, 0x64, 0x6f, 0x67, 0x41, 0x70, 0x69, 0x6b, 0x65, 0x79, 0x10, 0x93, 0x08, 0x12, 0x11, 0x0a, 0x0c, 0x53, 0x68, 0x6f, 0x70, 0x69, 0x66, 0x79, 0x4f, - 0x41, 0x75, 0x74, 0x68, 0x10, 0x94, 0x08, 0x42, 0x41, 0x5a, 0x3f, 0x67, 0x69, 0x74, 0x68, 0x75, - 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73, 0x65, 0x63, - 0x75, 0x72, 0x69, 0x74, 0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68, 0x6f, 0x67, - 0x2f, 0x76, 0x33, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x64, 0x65, 0x74, 0x65, 0x63, - 0x74, 0x6f, 0x72, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x33, + 0x41, 0x75, 0x74, 0x68, 0x10, 0x94, 0x08, 0x12, 0x18, 0x0a, 0x13, 0x42, 0x69, 0x74, 0x62, 0x75, + 0x63, 0x6b, 0x65, 0x74, 0x44, 0x61, 0x74, 0x61, 0x43, 0x65, 0x6e, 0x74, 0x65, 0x72, 0x10, 0x95, + 0x08, 0x42, 0x41, 0x5a, 0x3f, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, + 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x2f, + 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68, 0x6f, 0x67, 0x2f, 0x76, 0x33, 0x2f, 0x70, 0x6b, + 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x5f, 0x74, 0x79, + 0x70, 0x65, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/proto/detector_type.proto b/proto/detector_type.proto index 8575d10a4457..6d096c9b0ad3 100644 --- a/proto/detector_type.proto +++ b/proto/detector_type.proto @@ -1046,4 +1046,5 @@ enum DetectorType { ArtifactoryReferenceToken = 1042; DatadogApikey = 1043; ShopifyOAuth = 1044; + BitbucketDataCenter=1045; }