bdfr: patch imgur headers
This commit is contained in:
parent
d6095491f7
commit
0e5c473aec
2 changed files with 37 additions and 0 deletions
|
@ -14,6 +14,7 @@ pkgs.python3.pkgs.buildPythonApplication rec {
|
||||||
|
|
||||||
patches = [
|
patches = [
|
||||||
../patches/bulk-downloader-for-reddit_recursion-limit.patch
|
../patches/bulk-downloader-for-reddit_recursion-limit.patch
|
||||||
|
../patches/bulk-downloader-for-reddit_imgur-headers.patch
|
||||||
];
|
];
|
||||||
|
|
||||||
nativeBuildInputs = with pkgs.python3.pkgs; [
|
nativeBuildInputs = with pkgs.python3.pkgs; [
|
||||||
|
|
36
patches/bulk-downloader-for-reddit_imgur-headers.patch
Normal file
36
patches/bulk-downloader-for-reddit_imgur-headers.patch
Normal file
|
@ -0,0 +1,36 @@
|
||||||
|
Subject: Set custom headers for imgur
|
||||||
|
|
||||||
|
---
|
||||||
|
bdfr/resource.py | 8 ++++++++
|
||||||
|
1 file changed, 8 insertions(+)
|
||||||
|
|
||||||
|
--- a/bdfr/resource.py
|
||||||
|
+++ b/bdfr/resource.py
|
||||||
|
@@ -1,10 +1,12 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
|
||||||
|
+import fnmatch
|
||||||
|
import hashlib
|
||||||
|
import logging
|
||||||
|
import re
|
||||||
|
import time
|
||||||
|
import urllib.parse
|
||||||
|
+from urllib.parse import urlparse
|
||||||
|
from collections.abc import Callable
|
||||||
|
from typing import Optional
|
||||||
|
|
||||||
|
@@ -66,6 +68,12 @@ class Resource:
|
||||||
|
@staticmethod
|
||||||
|
def http_download(url: str, download_parameters: dict) -> Optional[bytes]:
|
||||||
|
headers = download_parameters.get("headers")
|
||||||
|
+ domain = urlparse(url).hostname
|
||||||
|
+ if fnmatch.fnmatch(domain, "*.imgur.com"):
|
||||||
|
+ headers = {
|
||||||
|
+ "user-agent": "curl/8.1.1",
|
||||||
|
+ "accept": "*/*"
|
||||||
|
+ }
|
||||||
|
current_wait_time = 60
|
||||||
|
if "max_wait_time" in download_parameters:
|
||||||
|
max_wait_time = download_parameters["max_wait_time"]
|
||||||
|
--
|
||||||
|
2.43.1
|
Loading…
Reference in a new issue