From f7a338cdbe4c6f6d5fd173fad00bb4194606dc00 Mon Sep 17 00:00:00 2001 From: Oliphant Date: Thu, 11 May 2023 19:53:02 +0000 Subject: [PATCH] I Want Receipts Edition --- RUNNING.md | 15 ++++- config/100.percent.conf.toml | 4 +- config/council.tier0.conf.toml | 4 +- config/example-server-export.conf.toml | 2 +- config/ms-noblock.conf.toml | 4 +- config/ms-we-block.conf.toml | 4 +- config/pull.all.conf.toml | 76 ++++++++++++++++++++++++++ config/tier0.conf.toml | 4 +- config/tier1-threshold.conf.toml | 4 +- config/tier1.conf.toml | 4 +- config/tier2-threshold.conf.toml | 4 +- config/tier2.conf.toml | 4 +- config/tier3-threshold.conf.toml | 4 +- config/tier3.conf.toml | 4 +- fedisync.sh | 5 +- 15 files changed, 117 insertions(+), 25 deletions(-) create mode 100644 config/pull.all.conf.toml diff --git a/RUNNING.md b/RUNNING.md index c51004a..308f5c9 100644 --- a/RUNNING.md +++ b/RUNNING.md @@ -34,14 +34,27 @@ The list of blocklist sources can be found in the [/config/pull.conf.toml](/olip These initial sources are pulled down, with each of them being saved as an intermediate file. +> Note: If you want to include the public_description, modify the [/config/pull.all.conf.toml](/oliphant/blocklists/src/branch/main/config/pull.all.conf.toml) file. + The `fedisync.sh` script renames each of these source lists to a more friendly name (like mastodon.social.csv) The remaining steps are performed on these sourcefiles, not pulled from urls. -If you customize the list of sources in `pull.conf.toml` you should also update them in the various `config/*.conf.toml` files as needed. +If you customize the list of sources in `pull.conf.toml` or `pull.all.conf.toml` you should also update them in the various `config/*.conf.toml` files as needed. You will likely also need to customize the `fedisync.sh` to your needs as well if you make these changes. +## Generate files with descriptions + +Edit the `fedisync.sh` file, and where you see as follows: + +``` +PULLCONFIG="/opt/fediblockhole/config/pull.conf.toml" # default, pull sources WITHOUT public_description +#PULLCONFIG="/opt/fediblockhole/config/pull.all.conf.toml" # pull down all sources WITH public_description +``` + +comment out the first line and uncomment the second. Then run `.fedisync.sh` as normal. The resulting files will have public_comment populated. + ## Pushing to Your Own Server If you want to push these blocks to your own server, that's an option, too. diff --git a/config/100.percent.conf.toml b/config/100.percent.conf.toml index 7340dd4..b2288fc 100644 --- a/config/100.percent.conf.toml +++ b/config/100.percent.conf.toml @@ -64,8 +64,8 @@ merge_threshold = 100 ## Set which fields we import ## 'domain' and 'severity' are always imported, these are additional ## -import_fields = ['reject_media', 'reject_reports', 'obfuscate'] -#import_fields = ['public_comment', 'reject_media', 'reject_reports', 'obfuscate'] +#import_fields = ['reject_media', 'reject_reports', 'obfuscate'] +import_fields = ['public_comment', 'reject_media', 'reject_reports', 'obfuscate'] ## Set which fields we export ## 'domain' and 'severity' are always exported, these are additional diff --git a/config/council.tier0.conf.toml b/config/council.tier0.conf.toml index 8fb65d0..f295814 100644 --- a/config/council.tier0.conf.toml +++ b/config/council.tier0.conf.toml @@ -64,8 +64,8 @@ merge_threshold = 80 ## Set which fields we import ## 'domain' and 'severity' are always imported, these are additional ## -import_fields = ['reject_media', 'reject_reports', 'obfuscate'] -#import_fields = ['public_comment', 'reject_media', 'reject_reports', 'obfuscate'] +#import_fields = ['reject_media', 'reject_reports', 'obfuscate'] +import_fields = ['public_comment', 'reject_media', 'reject_reports', 'obfuscate'] ## Set which fields we export ## 'domain' and 'severity' are always exported, these are additional diff --git a/config/example-server-export.conf.toml b/config/example-server-export.conf.toml index 82a40c8..61fe333 100644 --- a/config/example-server-export.conf.toml +++ b/config/example-server-export.conf.toml @@ -58,4 +58,4 @@ import_fields = ['public_comment', 'reject_media', 'reject_reports', 'obfuscate' ## Set which fields we export ## 'domain' and 'severity' are always exported, these are additional ## -export_fields = ['reject_media', 'reject_reports', 'obfuscate'] +export_fields = ['reject_media', 'reject_reports', 'public_comment', 'obfuscate'] diff --git a/config/ms-noblock.conf.toml b/config/ms-noblock.conf.toml index 0ff31ea..6856e60 100644 --- a/config/ms-noblock.conf.toml +++ b/config/ms-noblock.conf.toml @@ -54,8 +54,8 @@ mergeplan = 'min' ## Set which fields we import ## 'domain' and 'severity' are always imported, these are additional ## -import_fields = ['reject_media', 'reject_reports', 'obfuscate'] -#import_fields = ['public_comment', 'reject_media', 'reject_reports', 'obfuscate'] +#import_fields = ['reject_media', 'reject_reports', 'obfuscate'] +import_fields = ['public_comment', 'reject_media', 'reject_reports', 'obfuscate'] ## Set which fields we export ## 'domain' and 'severity' are always exported, these are additional diff --git a/config/ms-we-block.conf.toml b/config/ms-we-block.conf.toml index 5d2a3e7..1ed7455 100644 --- a/config/ms-we-block.conf.toml +++ b/config/ms-we-block.conf.toml @@ -53,8 +53,8 @@ mergeplan = 'min' ## Set which fields we import ## 'domain' and 'severity' are always imported, these are additional ## -import_fields = ['reject_media', 'reject_reports', 'obfuscate'] -#import_fields = ['public_comment', 'reject_media', 'reject_reports', 'obfuscate'] +#import_fields = ['reject_media', 'reject_reports', 'obfuscate'] +import_fields = ['public_comment', 'reject_media', 'reject_reports', 'obfuscate'] ## Set which fields we export ## 'domain' and 'severity' are always exported, these are additional diff --git a/config/pull.all.conf.toml b/config/pull.all.conf.toml new file mode 100644 index 0000000..d28f91e --- /dev/null +++ b/config/pull.all.conf.toml @@ -0,0 +1,76 @@ +## Pull Trusted Sources - With public_description + +# List of instances to read blocklists from. +# If the instance makes its blocklist public, no authorization token is needed. +# Otherwise, `token` is a Bearer token authorised to read domain_blocks. +# If `admin` = True, use the more detailed admin API, which requires a token with a +# higher level of authorization. +# If `import_fields` are provided, only import these fields from the instance. +# Overrides the global `import_fields` setting. +blocklist_instance_sources = [ + { domain = 'mastodon.social' }, + { domain = 'mastodon.online' }, + { domain = 'union.place' }, + { domain = 'sunny.garden' }, + { domain = 'toot.wales' }, + { domain = 'mastodon.art'}, + { domain = 'rage.love' }, + { domain = 'artisan.chat'}, + { domain = 'solarpunk.moe'}, +] + +# List of URLs to read csv blocklists from +# Format tells the parser which format to use when parsing the blocklist +# max_severity tells the parser to override any severities that are higher than this value +# import_fields tells the parser to only import that set of fields from a specific source +blocklist_url_sources = [ + # tier 0 + { url = 'https://seirdy.one/pb/tier0.csv', format = 'csv'}, + { url = 'https://raw.githubusercontent.com/gardenfence/blocklist/main/gardenfence-fediblocksync.csv', format = 'csv'}, + # other sources + { url = 'https://seirdy.one/pb/pleroma.envs.net.csv', format = 'csv' }, +] + +# assumes a git pull will proceed this check and bring down the most recent allowlist +allowlist_url_sources = [ + { url = 'file:///opt/fediblockhole/blocklists/__allowlist.csv', format = 'csv' }, +] + +# List of instances to write blocklist to +blocklist_instance_destinations = [ + # { domain = 'eigenmagic.net', token = '', max_followed_severity = 'silence'}, +] + +## Store a local copy of the remote blocklists after we fetch them +save_intermediate = true + +## Directory to store the local blocklist copies +savedir = '/opt/fediblockhole/blocklists/' + +## File to save the fully merged blocklist into +blocklist_savefile = '/opt/fediblockhole/blocklists/_unified_max_blocklist.csv' + +## Don't push blocklist to instances, even if they're defined above +no_push_instance = true + +## Don't fetch blocklists from URLs, even if they're defined above +# no_fetch_url = false + +## Don't fetch blocklists from instances, even if they're defined above +# no_fetch_instance = false + +## Set the mergeplan to use when dealing with overlaps between blocklists +# The default 'max' mergeplan will use the harshest severity block found for a domain. +# The 'min' mergeplan will use the lightest severity block found for a domain. +mergeplan = 'max' + +## Set which fields we import +## 'domain' and 'severity' are always imported, these are additional +## +#import_fields = ['reject_media', 'reject_reports', 'obfuscate'] +import_fields = ['public_comment', 'reject_media', 'reject_reports', 'obfuscate'] + +## Set which fields we export +## 'domain' and 'severity' are always exported, these are additional +## +export_fields = ['reject_media', 'reject_reports', 'public_comment', 'obfuscate'] diff --git a/config/tier0.conf.toml b/config/tier0.conf.toml index e087eab..6c4424a 100644 --- a/config/tier0.conf.toml +++ b/config/tier0.conf.toml @@ -55,8 +55,8 @@ mergeplan = 'max' ## Set which fields we import ## 'domain' and 'severity' are always imported, these are additional ## -import_fields = ['reject_media', 'reject_reports', 'obfuscate'] -#import_fields = ['public_comment', 'reject_media', 'reject_reports', 'obfuscate'] +#import_fields = ['reject_media', 'reject_reports', 'obfuscate'] +import_fields = ['public_comment', 'reject_media', 'reject_reports', 'obfuscate'] ## Set which fields we export ## 'domain' and 'severity' are always exported, these are additional diff --git a/config/tier1-threshold.conf.toml b/config/tier1-threshold.conf.toml index d3e6977..7246147 100644 --- a/config/tier1-threshold.conf.toml +++ b/config/tier1-threshold.conf.toml @@ -78,8 +78,8 @@ merge_threshold = 50 ## Set which fields we import ## 'domain' and 'severity' are always imported, these are additional ## -import_fields = ['reject_media', 'reject_reports', 'obfuscate'] -#import_fields = ['public_comment', 'reject_media', 'reject_reports', 'obfuscate'] +#import_fields = ['reject_media', 'reject_reports', 'obfuscate'] +import_fields = ['public_comment', 'reject_media', 'reject_reports', 'obfuscate'] ## Set which fields we export ## 'domain' and 'severity' are always exported, these are additional diff --git a/config/tier1.conf.toml b/config/tier1.conf.toml index 5c77cc1..a2f6032 100644 --- a/config/tier1.conf.toml +++ b/config/tier1.conf.toml @@ -56,8 +56,8 @@ mergeplan = 'min' ## Set which fields we import ## 'domain' and 'severity' are always imported, these are additional ## -import_fields = ['reject_media', 'reject_reports', 'obfuscate'] -#import_fields = ['public_comment', 'reject_media', 'reject_reports', 'obfuscate'] +#import_fields = ['reject_media', 'reject_reports', 'obfuscate'] +import_fields = ['public_comment', 'reject_media', 'reject_reports', 'obfuscate'] ## Set which fields we export ## 'domain' and 'severity' are always exported, these are additional diff --git a/config/tier2-threshold.conf.toml b/config/tier2-threshold.conf.toml index c2640d0..8752bc0 100644 --- a/config/tier2-threshold.conf.toml +++ b/config/tier2-threshold.conf.toml @@ -78,8 +78,8 @@ merge_threshold = 33 ## Set which fields we import ## 'domain' and 'severity' are always imported, these are additional ## -import_fields = ['reject_media', 'reject_reports', 'obfuscate'] -#import_fields = ['public_comment', 'reject_media', 'reject_reports', 'obfuscate'] +#import_fields = ['reject_media', 'reject_reports', 'obfuscate'] +import_fields = ['public_comment', 'reject_media', 'reject_reports', 'obfuscate'] ## Set which fields we export ## 'domain' and 'severity' are always exported, these are additional diff --git a/config/tier2.conf.toml b/config/tier2.conf.toml index 6ba0ad5..e5b1b0d 100644 --- a/config/tier2.conf.toml +++ b/config/tier2.conf.toml @@ -56,8 +56,8 @@ mergeplan = 'min' ## Set which fields we import ## 'domain' and 'severity' are always imported, these are additional ## -import_fields = ['reject_media', 'reject_reports', 'obfuscate'] -#import_fields = ['public_comment', 'reject_media', 'reject_reports', 'obfuscate'] +#import_fields = ['reject_media', 'reject_reports', 'obfuscate'] +import_fields = ['public_comment', 'reject_media', 'reject_reports', 'obfuscate'] ## Set which fields we export ## 'domain' and 'severity' are always exported, these are additional diff --git a/config/tier3-threshold.conf.toml b/config/tier3-threshold.conf.toml index a06ff9b..6625701 100644 --- a/config/tier3-threshold.conf.toml +++ b/config/tier3-threshold.conf.toml @@ -78,8 +78,8 @@ merge_threshold = 2 ## Set which fields we import ## 'domain' and 'severity' are always imported, these are additional ## -import_fields = ['reject_media', 'reject_reports', 'obfuscate'] -#import_fields = ['public_comment', 'reject_media', 'reject_reports', 'obfuscate'] +#import_fields = ['reject_media', 'reject_reports', 'obfuscate'] +import_fields = ['public_comment', 'reject_media', 'reject_reports', 'obfuscate'] ## Set which fields we export ## 'domain' and 'severity' are always exported, these are additional diff --git a/config/tier3.conf.toml b/config/tier3.conf.toml index 07e75ef..5463cb0 100644 --- a/config/tier3.conf.toml +++ b/config/tier3.conf.toml @@ -56,8 +56,8 @@ mergeplan = 'min' ## Set which fields we import ## 'domain' and 'severity' are always imported, these are additional ## -import_fields = ['reject_media', 'reject_reports', 'obfuscate'] -#import_fields = ['public_comment', 'reject_media', 'reject_reports', 'obfuscate'] +#import_fields = ['reject_media', 'reject_reports', 'obfuscate'] +import_fields = ['public_comment', 'reject_media', 'reject_reports', 'obfuscate'] ## Set which fields we export ## 'domain' and 'severity' are always exported, these are additional diff --git a/fedisync.sh b/fedisync.sh index 8a10ab1..0459bad 100755 --- a/fedisync.sh +++ b/fedisync.sh @@ -12,8 +12,11 @@ GIT=`which git` ${GIT} fetch #${GIT} fetchall +PULLCONFIG="/opt/fediblockhole/config/pull.conf.toml" # default, pull sources WITHOUT public_description +#PULLCONFIG="/opt/fediblockhole/config/pull.all.conf.toml" # pull down all sources WITH public_description + SYNC=`which fediblock-sync` -${SYNC} -c /opt/fediblockhole/config/pull.conf.toml # pull down all files, create max list +${SYNC} -c $PULLCONFIG # pull down all files, create max list # rename and remove intermediate files rm -f /opt/fediblockhole/blocklists/file:---opt-fediblockhole-blocklists-__allowlist.csv.csv