first commit

.gitignore

@@ -0,0 +1,7 @@
+/results.jsonl
+/prefix-list.txt
+/domain-list.txt
+/dns-list.txt
+/dns-backup-tool.iml
+/venv/
+/.idea/

install-macos.sh

@@ -0,0 +1,14 @@
+#!/bin/zsh
+
+brew install pkg-config intltool icu4c gettext jq virtualenv
+brew link icu4c gettext --force
+
+virtualenv -p /usr/local/bin/python3.11 venv
+source venv/bin/activate
+pip install --upgrade pip
+
+export PATH="/usr/local/opt/icu4c/bin:/usr/local/opt/icu4c/sbin:$PATH"
+export PKG_CONFIG_PATH="$PKG_CONFIG_PATH:/usr/local/opt/icu4c/lib/pkgconfig"
+pip install --no-binary=:pyicu: pyicu
+
+pip install -r requirements.txt

join.sh

@@ -0,0 +1,5 @@
+#!/bin/zsh
+
+join -t '' -j 2 suffix-list.txt dns-list.txt | sort -u > domain-list-n.txt
+tr < domain-list-n.txt -d '\000' > domain-list.txt
+rm domain-list-n.txt

requirements.txt

@@ -0,0 +1,33 @@
+asn1crypto==1.5.1
+async-timeout==4.0.3
+cert-human==1.0.7
+certifi==2024.7.4
+cffi==1.16.0
+chardet==3.0.4
+checkdmarc==4.3.1
+click==8.1.7
+cryptography==38.0.4
+dns-crawler==1.6.5
+dnspython==2.3.0
+ecdsa==0.18.0
+expiringdict==1.2.2
+forcediphttpsadapter==1.0.2
+geoip2==3.0.0
+hstspreload==2024.7.1
+idna==2.10
+maxminddb==2.6.2
+publicsuffix2==2.20191221
+pyaml==21.10.1
+pycparser==2.22
+PyICU==2.13.1
+pyleri==1.4.3
+pyOpenSSL==23.1.1
+python-magic==0.4.27
+PyYAML==6.0.1
+redis==4.5.4
+requests==2.23.0
+requests-toolbelt==0.10.1
+rq==1.13.0
+six==1.16.0
+timeout-decorator==0.5.0
+urllib3==1.25.2

run-dns-crawler.sh

@@ -0,0 +1,3 @@
+#!/bin/zsh
+
+dns-crawler domain-list.txt > results.jsonl