1
0
Fork 0

bdfr: patch imgur headers
All checks were successful
Build / build-amd64-linux (push) Successful in 37s
Build / build-arm64-linux (push) Successful in 59s

This commit is contained in:
Daniel Kempkens 2024-03-02 14:06:25 +01:00
parent d6095491f7
commit 0e5c473aec
Signed by: daniel
SSH key fingerprint: SHA256:Ks/MyhQYcPRQiwMKLAKquWCdCPe3JXlb1WttgnAoSeM
2 changed files with 37 additions and 0 deletions

View file

@ -14,6 +14,7 @@ pkgs.python3.pkgs.buildPythonApplication rec {
patches = [ patches = [
../patches/bulk-downloader-for-reddit_recursion-limit.patch ../patches/bulk-downloader-for-reddit_recursion-limit.patch
../patches/bulk-downloader-for-reddit_imgur-headers.patch
]; ];
nativeBuildInputs = with pkgs.python3.pkgs; [ nativeBuildInputs = with pkgs.python3.pkgs; [

View file

@ -0,0 +1,36 @@
Subject: Set custom headers for imgur
---
bdfr/resource.py | 8 ++++++++
1 file changed, 8 insertions(+)
--- a/bdfr/resource.py
+++ b/bdfr/resource.py
@@ -1,10 +1,12 @@
#!/usr/bin/env python3
+import fnmatch
import hashlib
import logging
import re
import time
import urllib.parse
+from urllib.parse import urlparse
from collections.abc import Callable
from typing import Optional
@@ -66,6 +68,12 @@ class Resource:
@staticmethod
def http_download(url: str, download_parameters: dict) -> Optional[bytes]:
headers = download_parameters.get("headers")
+ domain = urlparse(url).hostname
+ if fnmatch.fnmatch(domain, "*.imgur.com"):
+ headers = {
+ "user-agent": "curl/8.1.1",
+ "accept": "*/*"
+ }
current_wait_time = 60
if "max_wait_time" in download_parameters:
max_wait_time = download_parameters["max_wait_time"]
--
2.43.1