From e8d03850709d7820d89822b34c8a11aae31d5b93 Mon Sep 17 00:00:00 2001 From: MOOn Date: Tue, 2 May 2023 16:59:29 +0800 Subject: [PATCH] Initial commit --- .gitea/workflows/build.yaml | 70 +++++++ .gitignore | 3 + go.mod | 14 ++ go.sum | 27 +++ main.go | 392 ++++++++++++++++++++++++++++++++++++ 5 files changed, 506 insertions(+) create mode 100644 .gitea/workflows/build.yaml create mode 100644 .gitignore create mode 100644 go.mod create mode 100644 go.sum create mode 100644 main.go diff --git a/.gitea/workflows/build.yaml b/.gitea/workflows/build.yaml new file mode 100644 index 0000000..580b311 --- /dev/null +++ b/.gitea/workflows/build.yaml @@ -0,0 +1,70 @@ +name: Build geosite.dat +on: + workflow_dispatch: + schedule: + - cron: "30 21 * * *" + push: + branches: + - automation + paths-ignore: + - "**/README.md" +jobs: + build: + runs-on: ubuntu-latest + steps: + # - name: Compare latest tags and set variables + # run: | + # upstreamLatestTag=$(curl -sSL --connect-timeout 5 --retry 5 -H "Accept: application/vnd.github.v3+json" https://api.github.com/repos/v2fly/domain-list-community/releases/latest | grep "tag_name" | cut -d\" -f4) + # thisLatestTag=$(curl -sSL --connect-timeout 5 --retry 5 -H "Accept: application/vnd.github.v3+json" https://api.github.com/repos/${{ github.repository }}/releases/latest | grep "tag_name" | cut -d\" -f4) + # if [[ $upstreamLatestTag != $thisLatestTag ]]; then + # echo "NeedToSync=true" >> $GITHUB_ENV + # fi + # echo "RELEASE_NAME=$upstreamLatestTag" >> $GITHUB_ENV + # echo "TAG_NAME=$upstreamLatestTag" >> $GITHUB_ENV + # shell: bash + + - name: Checkout codebase + uses: https://github.com/actions/checkout@v3 + # if: ${{ env.NeedToSync }} + + - name: Checkout v2fly/domain-list-community + # if: ${{ env.NeedToSync }} + uses: https://github.com/actions/checkout@v3 + with: + repository: 'v2fly/domain-list-community' + ref: 'master' + path: 'domain-list-community' + token: ${{ secrets.GH_TOKEN }} + github-server-url: 'https://github.com' + + - name: Append attribute rules + # if: ${{ env.NeedToSync }} + run: | + echo "include:netflix" >> ./domain-list-community/data/streaming + + - name: Setup Go + uses: https://github.com/actions/setup-go@v4 + # if: ${{ env.NeedToSync }} + with: + go-version-file: ./go.mod + + - name: Set variables + run: | + echo "RELEASE_NAME=$(date +%Y%m%d%H%M%S)" >> $GITHUB_ENV + # echo "TAG_NAME=$(date +%Y%m%d%H%M%S)" >> $GITHUB_ENV + shell: bash + + - name: Get dependencies and run + # if: ${{ env.NeedToSync }} + run: | + go run ./ --datapath=./domain-list-community/data --exportlists=streaming + + - name: Git push assets to "release" branch + # if: ${{ env.NeedToSync }} + run: | + git init + git config --local user.name "gitea-actions[bot]" + git config --local user.email "gitea-actions[bot]@moon.re" + git add *.txt + git commit -m "${{ env.RELEASE_NAME }}" + git push diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..39b72eb --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ + +.DS_Store +**/.DS_Store diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..099fcf5 --- /dev/null +++ b/go.mod @@ -0,0 +1,14 @@ +module github.com/v2fly/domain-list-community + +go 1.19 + +require ( + github.com/v2fly/v2ray-core/v5 v5.4.1 + google.golang.org/protobuf v1.30.0 +) + +require ( + github.com/adrg/xdg v0.4.0 // indirect + github.com/golang/protobuf v1.5.2 // indirect + golang.org/x/sys v0.5.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..c588ece --- /dev/null +++ b/go.sum @@ -0,0 +1,27 @@ +github.com/adrg/xdg v0.4.0 h1:RzRqFcjH4nE5C6oTAxhBtoE2IRyjBSa62SCbyPidvls= +github.com/adrg/xdg v0.4.0/go.mod h1:N6ag73EX4wyxeaoeHctc1mas01KZgsj5tYiAIwqJE/E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= +github.com/v2fly/v2ray-core/v5 v5.4.1 h1:1l3KIFKoOlZkUp6D9MUlN/gz6aOEx09YfRVmnwnoGIQ= +github.com/v2fly/v2ray-core/v5 v5.4.1/go.mod h1:8JUFMS/1biOF9rWV7V5IU3NKU8GhCd442MqW3DgdFKw= +golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= +google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/main.go b/main.go new file mode 100644 index 0000000..05253b6 --- /dev/null +++ b/main.go @@ -0,0 +1,392 @@ +package main + +import ( + "bufio" + "errors" + "flag" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + + router "github.com/v2fly/v2ray-core/v5/app/router/routercommon" + "google.golang.org/protobuf/proto" +) + +var ( + dataPath = flag.String("datapath", "./data", "Path to your custom 'data' directory") + outputName = flag.String("outputname", "dlc.dat", "Name of the generated dat file") + outputDir = flag.String("outputdir", "./", "Directory to place all generated files") + exportLists = flag.String("exportlists", "", "Lists to be flattened and exported in plaintext format, separated by ',' comma") +) + +type Entry struct { + Type string + Value string + Attrs []*router.Domain_Attribute +} + +type List struct { + Name string + Entry []Entry +} + +type ParsedList struct { + Name string + Inclusion map[string]bool + Entry []Entry +} + +func (l *ParsedList) toPlainText(listName string) error { + var entryBytes []byte + for _, entry := range l.Entry { + var attrString string + if entry.Attrs != nil { + for _, attr := range entry.Attrs { + attrString += "@" + attr.GetKey() + "," + } + attrString = strings.TrimRight(":"+attrString, ",") + } + // Entry output format is: type:domain.tld:@attr1,@attr2 + entryBytes = append(entryBytes, []byte(entry.Type+":"+entry.Value+attrString+"\n")...) + } + if err := ioutil.WriteFile(filepath.Join(*outputDir, listName+".txt"), entryBytes, 0644); err != nil { + return fmt.Errorf(err.Error()) + } + return nil +} + +func (l *ParsedList) toProto() (*router.GeoSite, error) { + site := &router.GeoSite{ + CountryCode: l.Name, + } + for _, entry := range l.Entry { + switch entry.Type { + case "domain": + site.Domain = append(site.Domain, &router.Domain{ + Type: router.Domain_RootDomain, + Value: entry.Value, + Attribute: entry.Attrs, + }) + case "regexp": + site.Domain = append(site.Domain, &router.Domain{ + Type: router.Domain_Regex, + Value: entry.Value, + Attribute: entry.Attrs, + }) + case "keyword": + site.Domain = append(site.Domain, &router.Domain{ + Type: router.Domain_Plain, + Value: entry.Value, + Attribute: entry.Attrs, + }) + case "full": + site.Domain = append(site.Domain, &router.Domain{ + Type: router.Domain_Full, + Value: entry.Value, + Attribute: entry.Attrs, + }) + default: + return nil, errors.New("unknown domain type: " + entry.Type) + } + } + return site, nil +} + +func exportPlainTextList(list []string, refName string, pl *ParsedList) { + for _, listName := range list { + if strings.EqualFold(refName, listName) { + if err := pl.toPlainText(strings.ToLower(refName)); err != nil { + fmt.Println("Failed: ", err) + continue + } + fmt.Printf("'%s' has been generated successfully.\n", listName) + } + } +} + +func removeComment(line string) string { + idx := strings.Index(line, "#") + if idx == -1 { + return line + } + return strings.TrimSpace(line[:idx]) +} + +func parseDomain(domain string, entry *Entry) error { + kv := strings.Split(domain, ":") + if len(kv) == 1 { + entry.Type = "domain" + entry.Value = strings.ToLower(kv[0]) + return nil + } + + if len(kv) == 2 { + entry.Type = strings.ToLower(kv[0]) + entry.Value = strings.ToLower(kv[1]) + return nil + } + + return errors.New("Invalid format: " + domain) +} + +func parseAttribute(attr string) (*router.Domain_Attribute, error) { + var attribute router.Domain_Attribute + if len(attr) == 0 || attr[0] != '@' { + return &attribute, errors.New("invalid attribute: " + attr) + } + + // Trim attribute prefix `@` character + attr = attr[1:] + parts := strings.Split(attr, "=") + if len(parts) == 1 { + attribute.Key = strings.ToLower(parts[0]) + attribute.TypedValue = &router.Domain_Attribute_BoolValue{BoolValue: true} + } else { + attribute.Key = strings.ToLower(parts[0]) + intv, err := strconv.Atoi(parts[1]) + if err != nil { + return &attribute, errors.New("invalid attribute: " + attr + ": " + err.Error()) + } + attribute.TypedValue = &router.Domain_Attribute_IntValue{IntValue: int64(intv)} + } + return &attribute, nil +} + +func parseEntry(line string) (Entry, error) { + line = strings.TrimSpace(line) + parts := strings.Split(line, " ") + + var entry Entry + if len(parts) == 0 { + return entry, errors.New("empty entry") + } + + if err := parseDomain(parts[0], &entry); err != nil { + return entry, err + } + + for i := 1; i < len(parts); i++ { + attr, err := parseAttribute(parts[i]) + if err != nil { + return entry, err + } + entry.Attrs = append(entry.Attrs, attr) + } + + return entry, nil +} + +func Load(path string) (*List, error) { + file, err := os.Open(path) + if err != nil { + return nil, err + } + defer file.Close() + + list := &List{ + Name: strings.ToUpper(filepath.Base(path)), + } + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + line = removeComment(line) + if len(line) == 0 { + continue + } + entry, err := parseEntry(line) + if err != nil { + return nil, err + } + list.Entry = append(list.Entry, entry) + } + + return list, nil +} + +func isMatchAttr(Attrs []*router.Domain_Attribute, includeKey string) bool { + isMatch := false + mustMatch := true + matchName := includeKey + if strings.HasPrefix(includeKey, "!") { + isMatch = true + mustMatch = false + matchName = strings.TrimLeft(includeKey, "!") + } + + for _, Attr := range Attrs { + attrName := Attr.Key + if mustMatch { + if matchName == attrName { + isMatch = true + break + } + } else { + if matchName == attrName { + isMatch = false + break + } + } + } + return isMatch +} + +func createIncludeAttrEntrys(list *List, matchAttr *router.Domain_Attribute) []Entry { + newEntryList := make([]Entry, 0, len(list.Entry)) + matchName := matchAttr.Key + for _, entry := range list.Entry { + matched := isMatchAttr(entry.Attrs, matchName) + if matched { + newEntryList = append(newEntryList, entry) + } + } + return newEntryList +} + +func ParseList(list *List, ref map[string]*List) (*ParsedList, error) { + pl := &ParsedList{ + Name: list.Name, + Inclusion: make(map[string]bool), + } + entryList := list.Entry + for { + newEntryList := make([]Entry, 0, len(entryList)) + hasInclude := false + for _, entry := range entryList { + if entry.Type == "include" { + refName := strings.ToUpper(entry.Value) + if entry.Attrs != nil { + for _, attr := range entry.Attrs { + InclusionName := strings.ToUpper(refName + "@" + attr.Key) + if pl.Inclusion[InclusionName] { + continue + } + pl.Inclusion[InclusionName] = true + + refList := ref[refName] + if refList == nil { + return nil, errors.New(entry.Value + " not found.") + } + attrEntrys := createIncludeAttrEntrys(refList, attr) + if len(attrEntrys) != 0 { + newEntryList = append(newEntryList, attrEntrys...) + } + } + } else { + InclusionName := refName + if pl.Inclusion[InclusionName] { + continue + } + pl.Inclusion[InclusionName] = true + refList := ref[refName] + if refList == nil { + return nil, errors.New(entry.Value + " not found.") + } + newEntryList = append(newEntryList, refList.Entry...) + } + hasInclude = true + } else { + newEntryList = append(newEntryList, entry) + } + } + entryList = newEntryList + if !hasInclude { + break + } + } + pl.Entry = entryList + + return pl, nil +} + +func main() { + flag.Parse() + + dir := *dataPath + fmt.Println("Use domain lists in", dir) + + ref := make(map[string]*List) + err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if info.IsDir() { + return nil + } + list, err := Load(path) + if err != nil { + return err + } + ref[list.Name] = list + return nil + }) + if err != nil { + fmt.Println("Failed: ", err) + os.Exit(1) + } + + // Create output directory if not exist + if _, err := os.Stat(*outputDir); os.IsNotExist(err) { + if mkErr := os.MkdirAll(*outputDir, 0755); mkErr != nil { + fmt.Println("Failed: ", mkErr) + os.Exit(1) + } + } + + protoList := new(router.GeoSiteList) + var existList []string + for refName, list := range ref { + pl, err := ParseList(list, ref) + if err != nil { + fmt.Println("Failed: ", err) + os.Exit(1) + } + site, err := pl.toProto() + if err != nil { + fmt.Println("Failed: ", err) + os.Exit(1) + } + protoList.Entry = append(protoList.Entry, site) + + // Flatten and export plaintext list + if *exportLists != "" { + if existList != nil { + exportPlainTextList(existList, refName, pl) + } else { + exportedListSlice := strings.Split(*exportLists, ",") + for _, exportedListName := range exportedListSlice { + fileName := filepath.Join(dir, exportedListName) + _, err := os.Stat(fileName) + if err == nil || os.IsExist(err) { + existList = append(existList, exportedListName) + } else { + fmt.Printf("'%s' list does not exist in '%s' directory.\n", exportedListName, dir) + } + } + if existList != nil { + exportPlainTextList(existList, refName, pl) + } + } + } + } + + // Sort protoList so the marshaled list is reproducible + sort.SliceStable(protoList.Entry, func(i, j int) bool { + return protoList.Entry[i].CountryCode < protoList.Entry[j].CountryCode + }) + + protoBytes, err := proto.Marshal(protoList) + if err != nil { + fmt.Println("Failed:", err) + os.Exit(1) + } + if err := ioutil.WriteFile(filepath.Join(*outputDir, *outputName), protoBytes, 0644); err != nil { + fmt.Println("Failed: ", err) + os.Exit(1) + } else { + fmt.Println(*outputName, "has been generated successfully.") + } +}