bdfr: patch imgur headers
This commit is contained in:
parent
d6095491f7
commit
0e5c473aec
2 changed files with 37 additions and 0 deletions
|
@ -14,6 +14,7 @@ pkgs.python3.pkgs.buildPythonApplication rec {
|
|||
|
||||
patches = [
|
||||
../patches/bulk-downloader-for-reddit_recursion-limit.patch
|
||||
../patches/bulk-downloader-for-reddit_imgur-headers.patch
|
||||
];
|
||||
|
||||
nativeBuildInputs = with pkgs.python3.pkgs; [
|
||||
|
|
36
patches/bulk-downloader-for-reddit_imgur-headers.patch
Normal file
36
patches/bulk-downloader-for-reddit_imgur-headers.patch
Normal file
|
@ -0,0 +1,36 @@
|
|||
Subject: Set custom headers for imgur
|
||||
|
||||
---
|
||||
bdfr/resource.py | 8 ++++++++
|
||||
1 file changed, 8 insertions(+)
|
||||
|
||||
--- a/bdfr/resource.py
|
||||
+++ b/bdfr/resource.py
|
||||
@@ -1,10 +1,12 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
+import fnmatch
|
||||
import hashlib
|
||||
import logging
|
||||
import re
|
||||
import time
|
||||
import urllib.parse
|
||||
+from urllib.parse import urlparse
|
||||
from collections.abc import Callable
|
||||
from typing import Optional
|
||||
|
||||
@@ -66,6 +68,12 @@ class Resource:
|
||||
@staticmethod
|
||||
def http_download(url: str, download_parameters: dict) -> Optional[bytes]:
|
||||
headers = download_parameters.get("headers")
|
||||
+ domain = urlparse(url).hostname
|
||||
+ if fnmatch.fnmatch(domain, "*.imgur.com"):
|
||||
+ headers = {
|
||||
+ "user-agent": "curl/8.1.1",
|
||||
+ "accept": "*/*"
|
||||
+ }
|
||||
current_wait_time = 60
|
||||
if "max_wait_time" in download_parameters:
|
||||
max_wait_time = download_parameters["max_wait_time"]
|
||||
--
|
||||
2.43.1
|
Loading…
Reference in a new issue