feat: Postgres backend

This commit is contained in:
Daniel Kempkens 2023-08-14 01:18:55 +02:00
parent c819ff2f20
commit 277bee4dc4
Signed by: daniel
SSH key fingerprint: SHA256:Ks/MyhQYcPRQiwMKLAKquWCdCPe3JXlb1WttgnAoSeM
26 changed files with 708 additions and 105 deletions

View file

@ -4,5 +4,5 @@
"{config,lib,test}/**/*.{ex,exs}" "{config,lib,test}/**/*.{ex,exs}"
], ],
line_length: 120, line_length: 120,
import_deps: [:plug] import_deps: [:ecto, :ecto_sql, :plug]
] ]

1
.gitignore vendored
View file

@ -19,6 +19,7 @@ erl_crash.dump
# nix # nix
/.direnv /.direnv
/.elixir_ls /.elixir_ls
.pre-commit-config.yaml
# Exclude releases # Exclude releases
/rel/* /rel/*

View file

@ -1,7 +1,16 @@
import Config import Config
config :bdfr_browser,
ecto_repos: [BdfrBrowser.Repo]
config :bdfr_browser, BdfrBrowser.Repo,
migration_primary_key: [name: :id, type: :string],
migration_foreign_key: [column: :id, type: :string]
config :logger, config :logger,
backends: [], backends: [],
level: :warning, level: :info,
handle_otp_reports: false, handle_otp_reports: false,
handle_sasl_reports: false handle_sasl_reports: false
import_config "#{Mix.env()}.exs"

5
config/dev.exs Normal file
View file

@ -0,0 +1,5 @@
import Config
config :logger,
backends: [:console],
level: :debug

1
config/prod.exs Normal file
View file

@ -0,0 +1 @@
import Config

View file

@ -4,3 +4,9 @@ config :bdfr_browser,
base_directory: System.get_env("BDFR_BROWSER_BASE_DIRECTORY", "/nonexistant"), base_directory: System.get_env("BDFR_BROWSER_BASE_DIRECTORY", "/nonexistant"),
http_ip: to_charlist(System.get_env("BDFR_BROWSER_HTTP_IP", "127.0.0.1")), http_ip: to_charlist(System.get_env("BDFR_BROWSER_HTTP_IP", "127.0.0.1")),
http_port: String.to_integer(System.get_env("BDFR_BROWSER_HTTP_PORT", "4040")) http_port: String.to_integer(System.get_env("BDFR_BROWSER_HTTP_PORT", "4040"))
config :bdfr_browser, BdfrBrowser.Repo,
database: System.get_env("BDFR_BROWSER_REPO_DATABASE", "postgres"),
username: System.get_env("BDFR_BROWSER_REPO_USER", "bdfr-browser"),
password: System.get_env("BDFR_BROWSER_REPO_PASSWORD", ""),
hostname: System.get_env("BDFR_BROWSER_REPO_HOST", "localhost")

1
config/test.exs Normal file
View file

@ -0,0 +1 @@
import Config

View file

@ -1,15 +1,31 @@
{ {
"nodes": { "nodes": {
"flake-compat": {
"flake": false,
"locked": {
"lastModified": 1673956053,
"narHash": "sha256-4gtG9iQuiKITOjNQQeQIpoIB6b16fm+504Ch3sNKLd8=",
"owner": "edolstra",
"repo": "flake-compat",
"rev": "35bb57c0c8d8b62bbfd284272c928ceb64ddbde9",
"type": "github"
},
"original": {
"owner": "edolstra",
"repo": "flake-compat",
"type": "github"
}
},
"flake-parts": { "flake-parts": {
"inputs": { "inputs": {
"nixpkgs-lib": "nixpkgs-lib" "nixpkgs-lib": "nixpkgs-lib"
}, },
"locked": { "locked": {
"lastModified": 1683560683, "lastModified": 1690933134,
"narHash": "sha256-XAygPMN5Xnk/W2c1aW0jyEa6lfMDZWlQgiNtmHXytPc=", "narHash": "sha256-ab989mN63fQZBFrkk4Q8bYxQCktuHmBIBqUG1jl6/FQ=",
"owner": "hercules-ci", "owner": "hercules-ci",
"repo": "flake-parts", "repo": "flake-parts",
"rev": "006c75898cf814ef9497252b022e91c946ba8e17", "rev": "59cf3f1447cfc75087e7273b04b31e689a8599fb",
"type": "github" "type": "github"
}, },
"original": { "original": {
@ -18,13 +34,66 @@
"type": "github" "type": "github"
} }
}, },
"flake-root": {
"locked": {
"lastModified": 1680964220,
"narHash": "sha256-dIdTYcf+KW9a4pKHsEbddvLVSfR1yiAJynzg2x0nfWg=",
"owner": "srid",
"repo": "flake-root",
"rev": "f1c0b93d05bdbea6c011136ba1a135c80c5b326c",
"type": "github"
},
"original": {
"owner": "srid",
"repo": "flake-root",
"type": "github"
}
},
"flake-utils": {
"inputs": {
"systems": "systems"
},
"locked": {
"lastModified": 1685518550,
"narHash": "sha256-o2d0KcvaXzTrPRIo0kOLV0/QXHhDQ5DTi+OxcjO8xqY=",
"owner": "numtide",
"repo": "flake-utils",
"rev": "a1720a10a6cfe8234c0e93907ffe81be440f4cef",
"type": "github"
},
"original": {
"owner": "numtide",
"repo": "flake-utils",
"type": "github"
}
},
"gitignore": {
"inputs": {
"nixpkgs": [
"nixpkgs"
]
},
"locked": {
"lastModified": 1660459072,
"narHash": "sha256-8DFJjXG8zqoONA1vXtgeKXy68KdJL5UaXR8NtVMUbx8=",
"owner": "hercules-ci",
"repo": "gitignore.nix",
"rev": "a20de23b925fd8264fd7fad6454652e142fd7f73",
"type": "github"
},
"original": {
"owner": "hercules-ci",
"repo": "gitignore.nix",
"type": "github"
}
},
"nixpkgs": { "nixpkgs": {
"locked": { "locked": {
"lastModified": 1684242266, "lastModified": 1691853136,
"narHash": "sha256-uaCQ2k1bmojHKjWQngvnnnxQJMY8zi1zq527HdWgQf8=", "narHash": "sha256-wTzDsRV4HN8A2Sl0SVQY0q8ILs90CD43Ha//7gNZE+E=",
"owner": "NixOS", "owner": "NixOS",
"repo": "nixpkgs", "repo": "nixpkgs",
"rev": "7e0743a5aea1dc755d4b761daf75b20aa486fdad", "rev": "f0451844bbdf545f696f029d1448de4906c7f753",
"type": "github" "type": "github"
}, },
"original": { "original": {
@ -37,11 +106,11 @@
"nixpkgs-lib": { "nixpkgs-lib": {
"locked": { "locked": {
"dir": "lib", "dir": "lib",
"lastModified": 1682879489, "lastModified": 1690881714,
"narHash": "sha256-sASwo8gBt7JDnOOstnps90K1wxmVfyhsTPPNTGBPjjg=", "narHash": "sha256-h/nXluEqdiQHs1oSgkOOWF+j8gcJMWhwnZ9PFabN6q0=",
"owner": "NixOS", "owner": "NixOS",
"repo": "nixpkgs", "repo": "nixpkgs",
"rev": "da45bf6ec7bbcc5d1e14d3795c025199f28e0de0", "rev": "9e1960bc196baf6881340d53dccb203a951745a2",
"type": "github" "type": "github"
}, },
"original": { "original": {
@ -52,10 +121,107 @@
"type": "github" "type": "github"
} }
}, },
"nixpkgs-stable": {
"locked": {
"lastModified": 1685801374,
"narHash": "sha256-otaSUoFEMM+LjBI1XL/xGB5ao6IwnZOXc47qhIgJe8U=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "c37ca420157f4abc31e26f436c1145f8951ff373",
"type": "github"
},
"original": {
"owner": "NixOS",
"ref": "nixos-23.05",
"repo": "nixpkgs",
"type": "github"
}
},
"pre-commit-hooks-nix": {
"inputs": {
"flake-compat": "flake-compat",
"flake-utils": "flake-utils",
"gitignore": [
"gitignore"
],
"nixpkgs": [
"nixpkgs"
],
"nixpkgs-stable": "nixpkgs-stable"
},
"locked": {
"lastModified": 1691747570,
"narHash": "sha256-J3fnIwJtHVQ0tK2JMBv4oAmII+1mCdXdpeCxtIsrL2A=",
"owner": "cachix",
"repo": "pre-commit-hooks.nix",
"rev": "c5ac3aa3324bd8aebe8622a3fc92eeb3975d317a",
"type": "github"
},
"original": {
"owner": "cachix",
"repo": "pre-commit-hooks.nix",
"type": "github"
}
},
"process-compose-flake": {
"locked": {
"lastModified": 1691182187,
"narHash": "sha256-dmHKgFXstdfX7nfnpbzR5H4DWGdWo610zsW9BCtI2WQ=",
"owner": "Platonic-Systems",
"repo": "process-compose-flake",
"rev": "bacdaf54ffe3a2c1734fd973a95e6b39b1560c2e",
"type": "github"
},
"original": {
"owner": "Platonic-Systems",
"repo": "process-compose-flake",
"type": "github"
}
},
"root": { "root": {
"inputs": { "inputs": {
"flake-parts": "flake-parts", "flake-parts": "flake-parts",
"nixpkgs": "nixpkgs" "flake-root": "flake-root",
"gitignore": "gitignore",
"nixpkgs": "nixpkgs",
"pre-commit-hooks-nix": "pre-commit-hooks-nix",
"process-compose-flake": "process-compose-flake",
"treefmt-nix": "treefmt-nix"
}
},
"systems": {
"locked": {
"lastModified": 1681028828,
"narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
"owner": "nix-systems",
"repo": "default",
"rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
"type": "github"
},
"original": {
"owner": "nix-systems",
"repo": "default",
"type": "github"
}
},
"treefmt-nix": {
"inputs": {
"nixpkgs": [
"nixpkgs"
]
},
"locked": {
"lastModified": 1691833704,
"narHash": "sha256-ASGhgGduEgcD3gQZhGr8xtmZ3PlVY+m2HuPnIZDbu78=",
"owner": "numtide",
"repo": "treefmt-nix",
"rev": "19dee4bf6001849006a63f3435247316b0488e99",
"type": "github"
},
"original": {
"owner": "numtide",
"repo": "treefmt-nix",
"type": "github"
} }
} }
}, },

168
flake.nix
View file

@ -1,47 +1,193 @@
{ {
description = "bdfr-browser development environment"; description = "bdfr-browser development environment";
inputs = { inputs = {
nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable"; nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable";
# Tools
flake-parts.url = "github:hercules-ci/flake-parts"; flake-parts.url = "github:hercules-ci/flake-parts";
flake-root.url = "github:srid/flake-root";
treefmt-nix = {
url = "github:numtide/treefmt-nix";
inputs.nixpkgs.follows = "nixpkgs";
}; };
outputs = inputs@{ flake-parts, ... }: gitignore = {
url = "github:hercules-ci/gitignore.nix";
inputs.nixpkgs.follows = "nixpkgs";
};
pre-commit-hooks-nix = {
url = "github:cachix/pre-commit-hooks.nix";
inputs.nixpkgs.follows = "nixpkgs";
inputs.gitignore.follows = "gitignore";
};
process-compose-flake.url = "github:Platonic-Systems/process-compose-flake";
};
outputs = inputs@{ flake-parts, gitignore, ... }:
flake-parts.lib.mkFlake { inherit inputs; } { flake-parts.lib.mkFlake { inherit inputs; } {
systems = [ "aarch64-darwin" "x86_64-linux" "aarch64-linux" ]; systems = [ "aarch64-darwin" "x86_64-linux" "aarch64-linux" ];
perSystem = { pkgs, lib, self', ... }: imports = [
inputs.flake-root.flakeModule
inputs.treefmt-nix.flakeModule
inputs.pre-commit-hooks-nix.flakeModule
inputs.process-compose-flake.flakeModule
];
perSystem = { pkgs, lib, config, self', ... }:
let let
pname = "bdfr-browser"; pname = "bdfr-browser";
version = "0.0.1"; version = "0.0.1";
erlang = pkgs.beam.interpreters.erlangR25; erlang = pkgs.beam.interpreters.erlangR26;
beamPackages = pkgs.beam.packagesWith erlang; beamPackagesPrev = pkgs.beam.packagesWith erlang;
elixir = beamPackages.elixir_1_14; elixir = beamPackagesPrev.elixir_1_15;
beamPackages = beamPackagesPrev // rec {
inherit erlang elixir;
hex = beamPackagesPrev.hex.override { inherit elixir; };
buildMix = beamPackagesPrev.buildMix.override { inherit elixir erlang hex; };
mixRelease = beamPackagesPrev.mixRelease.override { inherit erlang elixir; };
};
postgres = pkgs.postgresql_15;
inherit (pkgs.stdenv) isDarwin; inherit (pkgs.stdenv) isDarwin;
inherit (gitignore.lib) gitignoreSource;
in in
{ {
treefmt = {
inherit (config.flake-root) projectRootFile;
flakeCheck = false;
programs = {
mix-format = {
enable = true;
package = elixir;
};
nixpkgs-fmt.enable = true;
};
};
pre-commit = {
check.enable = false;
settings = {
excludes = [ "mix.nix" ];
hooks = {
deadnix.enable = true;
statix.enable = true;
treefmt.enable = true;
};
};
};
devShells.default = pkgs.mkShell { devShells.default = pkgs.mkShell {
packages = (with pkgs; [ name = pname;
nativeBuildInputs = [
erlang erlang
elixir elixir
postgres
beamPackages.elixir-ls ] ++ lib.optionals isDarwin (with pkgs.darwin.apple_sdk.frameworks;
mix2nix [
]) ++ lib.optionals isDarwin (with pkgs.darwin.apple_sdk.frameworks; [
CoreFoundation CoreFoundation
CoreServices CoreServices
]); ]);
packages = [
pkgs.mix2nix
self'.packages.bdfr-browser-dev
];
inputsFrom = [
config.flake-root.devShell
config.treefmt.build.devShell
config.pre-commit.devShell
];
ERL_INCLUDE_PATH = "${erlang}/lib/erlang/usr/include"; ERL_INCLUDE_PATH = "${erlang}/lib/erlang/usr/include";
TREEFMT_CONFIG_FILE = config.treefmt.build.configFile;
}; };
packages.default = beamPackages.mixRelease { packages.default = beamPackages.mixRelease {
inherit pname version; inherit pname version;
src = ./.; src = gitignoreSource ./.;
mixNixDeps = import ./mix.nix { inherit lib beamPackages; }; mixNixDeps = import ./mix.nix { inherit lib beamPackages; };
}; };
process-compose."${pname}-dev" =
let
db-host = "127.0.0.1";
db-user = "bdfr-browser";
in
{
port = 18808;
settings = {
environment = {
BDFR_BROWSER_BASE_DIRECTORY = "/Volumes/MediaScraper/Reddit";
BDFR_BROWSER_REPO_USER = db-user;
BDFR_BROWSER_REPO_HOST = db-host;
};
processes = {
db-init.command = ''
if [ ! -d "$PWD/.direnv/postgres/data" ]; then
echo "Initializing database ..."
mkdir -p "$PWD/.direnv/postgres"
${postgres}/bin/initdb --username ${db-user} --pgdata "$PWD/.direnv/postgres/data" --auth trust
else
echo "Database already initialized"
fi
'';
db = {
command = "${postgres}/bin/postgres -D $PWD/.direnv/postgres/data";
depends_on.db-init.condition = "process_completed_successfully";
readiness_probe.exec.command = "PGCONNECT_TIMEOUT=1 ${postgres}/bin/psql -h ${db-host} -U ${db-user} -l";
};
app-setup.command = ''
mix local.hex --if-missing --force
mix local.rebar --force
mix deps.get
'';
app-compile = {
command = "mix release --overwrite";
depends_on.app-setup.condition = "process_completed_successfully";
};
app = {
command = "$PWD/_build/dev/rel/bdfr_browser/bin/bdfr_browser start";
depends_on = {
db.condition = "process_healthy";
app-compile.condition = "process_completed_successfully";
};
readiness_probe.http_get = {
host = "127.0.0.1";
port = 4040;
path = "/_ping";
};
};
};
};
};
}; };
}; };
} }

View file

@ -7,10 +7,14 @@ defmodule BdfrBrowser.Application do
@impl true @impl true
def start(_type, _args) do def start(_type, _args) do
repos = Application.fetch_env!(:bdfr_browser, :ecto_repos)
{:ok, http_ip} = :inet.parse_address(Application.fetch_env!(:bdfr_browser, :http_ip)) {:ok, http_ip} = :inet.parse_address(Application.fetch_env!(:bdfr_browser, :http_ip))
http_port = Application.fetch_env!(:bdfr_browser, :http_port) http_port = Application.fetch_env!(:bdfr_browser, :http_port)
children = [ children = [
{Ecto.Migrator, repos: repos, skip: System.get_env("SKIP_MIGRATIONS") == "true"},
BdfrBrowser.Repo,
BdfrBrowser.Importer,
{Plug.Cowboy, scheme: :http, plug: BdfrBrowser.HTTP.Plug, options: [ip: http_ip, port: http_port]}, {Plug.Cowboy, scheme: :http, plug: BdfrBrowser.HTTP.Plug, options: [ip: http_ip, port: http_port]},
:systemd.ready() :systemd.ready()
] ]

View file

@ -0,0 +1,19 @@
defmodule BdfrBrowser.Comment do
use Ecto.Schema
alias BdfrBrowser.Post
@primary_key {:id, :string, autogenerate: false}
@foreign_key_type :string
schema "comments" do
field :author, :string
field :body, :string
field :score, :integer
field :posted_at, :utc_datetime
belongs_to :post, Post
belongs_to :parent, __MODULE__
has_many :children, __MODULE__, foreign_key: :parent_id
end
end

View file

@ -1,12 +1,14 @@
defmodule BdfrBrowser.HTTP.Plug do defmodule BdfrBrowser.HTTP.Plug do
use Plug.Router use Plug.Router
alias BdfrBrowser.{Repo, Post, Subreddit}
plug :match plug :match
plug :dispatch plug :dispatch
get "/" do get "/" do
tpl_params = [ tpl_params = [
subreddits: list_folders(sort: :asc) subreddits: Subreddit.names() |> Repo.all()
] ]
tpl_file = Application.app_dir(:bdfr_browser, "priv/templates/http/index.eex") tpl_file = Application.app_dir(:bdfr_browser, "priv/templates/http/index.eex")
@ -18,9 +20,11 @@ defmodule BdfrBrowser.HTTP.Plug do
end end
get "/r/:subreddit" do get "/r/:subreddit" do
subreddit_record = Repo.get_by(Subreddit, name: subreddit)
tpl_params = [ tpl_params = [
subreddit: subreddit, subreddit: subreddit,
dates: list_folders(paths: [subreddit]) dates: subreddit_record |> Post.date_listing() |> Repo.all()
] ]
tpl_file = Application.app_dir(:bdfr_browser, "priv/templates/http/subreddit.eex") tpl_file = Application.app_dir(:bdfr_browser, "priv/templates/http/subreddit.eex")
@ -32,10 +36,12 @@ defmodule BdfrBrowser.HTTP.Plug do
end end
get "/r/:subreddit/:date" do get "/r/:subreddit/:date" do
subreddit_record = Repo.get_by(Subreddit, name: subreddit)
tpl_params = [ tpl_params = [
subreddit: subreddit, subreddit: subreddit,
date: date, date: date,
posts: read_posts(paths: [subreddit, date], ext: ".json") posts: subreddit_record |> Post.during_month(date) |> Repo.all()
] ]
tpl_file = Application.app_dir(:bdfr_browser, "priv/templates/http/subreddit_posts.eex") tpl_file = Application.app_dir(:bdfr_browser, "priv/templates/http/subreddit_posts.eex")
@ -46,12 +52,14 @@ defmodule BdfrBrowser.HTTP.Plug do
|> send_resp(200, content) |> send_resp(200, content)
end end
get "/r/:subreddit/:date/:post" do get "/r/:subreddit/:date/:id" do
post_record = Post |> Repo.get(id) |> Repo.preload(comments: :children)
tpl_params = [ tpl_params = [
subreddit: subreddit, subreddit: subreddit,
date: date, date: date,
post: read_post(post, paths: [subreddit, date]), post: post_record,
media: post_media(post, paths: [subreddit, date]), media: post_media(post_record.filename, paths: [subreddit, date]),
comment_template: Application.app_dir(:bdfr_browser, "priv/templates/http/_comment.eex") comment_template: Application.app_dir(:bdfr_browser, "priv/templates/http/_comment.eex")
] ]
@ -73,66 +81,21 @@ defmodule BdfrBrowser.HTTP.Plug do
|> send_resp(200, media) |> send_resp(200, media)
end end
get "/_import" do
:ok = BdfrBrowser.Importer.background_import()
send_resp(conn, 200, "IMPORTING")
end
get "/_ping" do
send_resp(conn, 200, "PONG")
end
match _ do match _ do
send_resp(conn, 404, "Not Found") send_resp(conn, 404, "Not Found")
end end
# Helper # Helper
defp list_folders(args) do
paths = Keyword.get(args, :paths, [])
extname = Keyword.get(args, :ext, "")
sort = Keyword.get(args, :sort, :desc)
base_directory = Application.fetch_env!(:bdfr_browser, :base_directory)
[base_directory | paths]
|> Path.join()
|> File.ls!()
|> Enum.filter(fn s -> not String.starts_with?(s, ".") and Path.extname(s) == extname end)
|> Enum.sort_by(&String.downcase/1, sort)
end
defp read_posts(args) do
posts = list_folders(args)
sort = Keyword.get(args, :sort, :desc)
base_directory = Application.fetch_env!(:bdfr_browser, :base_directory)
post_dir = Path.join([base_directory | Keyword.fetch!(args, :paths)])
compact_posts =
for post <- posts do
{:ok, content} = [post_dir, post] |> Path.join() |> File.read!() |> Jason.decode()
%{
title: extract_title(content["title"]),
author: content["author"],
num_comments: content["num_comments"],
created_utc: content["created_utc"],
filename: Path.basename(post, ".json")
}
end
Enum.sort_by(compact_posts, fn p -> p.created_utc end, sort)
end
defp extract_title(title) do
if String.length(title) == 0 do
"Empty Title"
else
title
end
end
defp read_post(post, args) do
base_directory = Application.fetch_env!(:bdfr_browser, :base_directory)
post_dir = Path.join([base_directory | Keyword.fetch!(args, :paths)])
post_file = "#{post}.json"
{:ok, content} = [post_dir, post_file] |> Path.join() |> File.read!() |> Jason.decode(keys: :atoms)
content
end
defp post_media(post, args) do defp post_media(post, args) do
base_directory = Application.fetch_env!(:bdfr_browser, :base_directory) base_directory = Application.fetch_env!(:bdfr_browser, :base_directory)
post_dir = Path.join([base_directory | Keyword.fetch!(args, :paths)]) post_dir = Path.join([base_directory | Keyword.fetch!(args, :paths)])

View file

@ -0,0 +1,145 @@
defmodule BdfrBrowser.Importer do
require Logger
use GenServer
alias BdfrBrowser.{Comment, Post, Repo, Subreddit}
def start_link([]) do
GenServer.start_link(__MODULE__, [], name: __MODULE__)
end
def subreddits do
_ = Logger.info("Importing subreddits ...")
folders = list_folders(sort: :asc)
for folder <- folders do
%Subreddit{name: folder}
|> Repo.insert(
on_conflict: :nothing,
conflict_target: :name
)
end
end
def posts_and_comments do
_ = Logger.info("Importing posts and comments ...")
result =
for subreddit <- list_folders(sort: :asc) do
_ = Logger.info("Importing entries from `#{subreddit}' ...")
subreddit_record = Repo.get_by(Subreddit, name: subreddit)
for date <- list_folders(paths: [subreddit]) do
_ = Logger.debug("Importing entries from `#{subreddit}' on `#{date}' ...")
for post <- read_posts(paths: [subreddit, date], ext: ".json") do
_ = Logger.debug("Importing `#{post["id"]}' from `#{subreddit}' ...")
{:ok, post_record} = import_post(post, subreddit_record)
comment_records = for comment <- post["comments"], do: import_comment(comment, post_record, nil)
{post_record, List.flatten(comment_records)}
end
end
end
List.flatten(result)
end
def background_import do
GenServer.cast(__MODULE__, :background_import)
end
# Callbacks
@impl true
def init([]) do
{:ok, nil}
end
@impl true
def handle_cast(:background_import, state) do
_ = subreddits()
_ = posts_and_comments()
{:noreply, state}
end
# Helper
defp list_folders(args) do
paths = Keyword.get(args, :paths, [])
extname = Keyword.get(args, :ext, "")
sort = Keyword.get(args, :sort, :desc)
base_directory = Application.fetch_env!(:bdfr_browser, :base_directory)
[base_directory | paths]
|> Path.join()
|> File.ls!()
|> Enum.filter(fn s -> not String.starts_with?(s, ".") and Path.extname(s) == extname end)
|> Enum.sort_by(&String.downcase/1, sort)
end
defp read_posts(args) do
posts = list_folders(args)
sort = Keyword.get(args, :sort, :desc)
base_directory = Application.fetch_env!(:bdfr_browser, :base_directory)
post_dir = Path.join([base_directory | Keyword.fetch!(args, :paths)])
parsed_posts =
for post <- posts do
file_path = Path.join([post_dir, post])
parsed = file_path |> File.read!() |> Jason.decode!()
Map.put(parsed, "filename", post)
end
Enum.sort_by(parsed_posts, fn p -> p["created_utc"] end, sort)
end
defp import_post(post, subreddit) do
id = post["id"]
%Post{
id: id,
title: post["title"],
selftext: post["selftext"],
url: post["url"],
permalink: post["permalink"],
author: post["author"],
upvote_ratio: post["upvote_ratio"],
posted_at: DateTime.from_unix!(trunc(post["created_utc"])),
filename: Path.basename(post["filename"], ".json"),
subreddit: subreddit
}
|> Repo.insert(
on_conflict: [set: [id: id]],
conflict_target: :id
)
end
defp import_comment(comment, post, parent) do
id = comment["id"]
{:ok, parent} =
%Comment{
id: id,
author: comment["author"],
body: comment["body"],
score: comment["score"],
posted_at: DateTime.from_unix!(trunc(comment["created_utc"])),
post: post,
parent: parent
}
|> Repo.insert(
on_conflict: [set: [id: id]],
conflict_target: :id
)
children = for child <- comment["replies"], do: import_comment(child, post, parent)
[parent] ++ children
end
end

49
lib/bdfr_browser/post.ex Normal file
View file

@ -0,0 +1,49 @@
defmodule BdfrBrowser.Post do
use Ecto.Schema
import Ecto.Query, only: [from: 2]
alias BdfrBrowser.{Comment, Subreddit}
@primary_key {:id, :string, autogenerate: false}
schema "posts" do
field :title, :string
field :selftext, :string
field :url, :string
field :permalink, :string
field :author, :string
field :upvote_ratio, :float
field :posted_at, :utc_datetime
field :filename, :string
belongs_to :subreddit, Subreddit
has_many :comments, Comment
end
def date_listing(subreddit) do
from(p in __MODULE__,
select: fragment("to_char(?, 'YYYY-MM')", p.posted_at),
where: p.subreddit_id == ^subreddit.id,
distinct: true,
order_by: [desc: fragment("to_char(?, 'YYYY-MM')", p.posted_at)]
)
end
def during_month(subreddit, month_str) do
{:ok, d} = Date.from_iso8601("#{month_str}-01")
during_range(subreddit, Date.beginning_of_month(d), Date.end_of_month(d))
end
def during_range(subreddit, start_date, end_date) do
from(p in __MODULE__,
left_join: c in assoc(p, :comments),
select: %{id: p.id, title: p.title, author: p.author, posted_at: p.posted_at, num_comments: count(c.id)},
where:
p.subreddit_id == ^subreddit.id and type(p.posted_at, :date) >= ^start_date and
type(p.posted_at, :date) <= ^end_date,
order_by: [desc: p.posted_at],
group_by: p.id
)
end
end

5
lib/bdfr_browser/repo.ex Normal file
View file

@ -0,0 +1,5 @@
defmodule BdfrBrowser.Repo do
use Ecto.Repo,
otp_app: :bdfr_browser,
adapter: Ecto.Adapters.Postgres
end

View file

@ -0,0 +1,17 @@
defmodule BdfrBrowser.Subreddit do
use Ecto.Schema
import Ecto.Query, only: [from: 2]
alias BdfrBrowser.Post
schema "subreddits" do
field(:name, :string)
has_many(:posts, Post)
end
def names do
from(s in __MODULE__, select: s.name, order_by: [asc: s.name])
end
end

View file

@ -21,6 +21,8 @@ defmodule BdfrBrowser.MixProject do
defp deps do defp deps do
[ [
{:plug_cowboy, "~> 2.6"}, {:plug_cowboy, "~> 2.6"},
{:ecto_sql, "~> 3.10"},
{:postgrex, "~> 0.17"},
{:jason, "~> 1.4"}, {:jason, "~> 1.4"},
{:earmark, "~> 1.4"}, {:earmark, "~> 1.4"},
{:systemd, "~> 0.6"} {:systemd, "~> 0.6"}

View file

@ -2,14 +2,19 @@
"cowboy": {:hex, :cowboy, "2.10.0", "ff9ffeff91dae4ae270dd975642997afe2a1179d94b1887863e43f681a203e26", [:make, :rebar3], [{:cowlib, "2.12.1", [hex: :cowlib, repo: "hexpm", optional: false]}, {:ranch, "1.8.0", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "3afdccb7183cc6f143cb14d3cf51fa00e53db9ec80cdcd525482f5e99bc41d6b"}, "cowboy": {:hex, :cowboy, "2.10.0", "ff9ffeff91dae4ae270dd975642997afe2a1179d94b1887863e43f681a203e26", [:make, :rebar3], [{:cowlib, "2.12.1", [hex: :cowlib, repo: "hexpm", optional: false]}, {:ranch, "1.8.0", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "3afdccb7183cc6f143cb14d3cf51fa00e53db9ec80cdcd525482f5e99bc41d6b"},
"cowboy_telemetry": {:hex, :cowboy_telemetry, "0.4.0", "f239f68b588efa7707abce16a84d0d2acf3a0f50571f8bb7f56a15865aae820c", [:rebar3], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "7d98bac1ee4565d31b62d59f8823dfd8356a169e7fcbb83831b8a5397404c9de"}, "cowboy_telemetry": {:hex, :cowboy_telemetry, "0.4.0", "f239f68b588efa7707abce16a84d0d2acf3a0f50571f8bb7f56a15865aae820c", [:rebar3], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "7d98bac1ee4565d31b62d59f8823dfd8356a169e7fcbb83831b8a5397404c9de"},
"cowlib": {:hex, :cowlib, "2.12.1", "a9fa9a625f1d2025fe6b462cb865881329b5caff8f1854d1cbc9f9533f00e1e1", [:make, :rebar3], [], "hexpm", "163b73f6367a7341b33c794c4e88e7dbfe6498ac42dcd69ef44c5bc5507c8db0"}, "cowlib": {:hex, :cowlib, "2.12.1", "a9fa9a625f1d2025fe6b462cb865881329b5caff8f1854d1cbc9f9533f00e1e1", [:make, :rebar3], [], "hexpm", "163b73f6367a7341b33c794c4e88e7dbfe6498ac42dcd69ef44c5bc5507c8db0"},
"earmark": {:hex, :earmark, "1.4.38", "ba8fda946c259c6e8f6759d3647d448e9216e2c0afed8c6ae7f8ce1f7072a497", [:mix], [{:earmark_parser, "~> 1.4.32", [hex: :earmark_parser, repo: "hexpm", optional: false]}], "hexpm", "f938e30de4167e7d8f3bf588b01dc041138278dda1e5a13fb9ec89b43dd5ec7f"}, "db_connection": {:hex, :db_connection, "2.5.0", "bb6d4f30d35ded97b29fe80d8bd6f928a1912ca1ff110831edcd238a1973652c", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "c92d5ba26cd69ead1ff7582dbb860adeedfff39774105a4f1c92cbb654b55aa2"},
"earmark_parser": {:hex, :earmark_parser, "1.4.32", "fa739a0ecfa34493de19426681b23f6814573faee95dfd4b4aafe15a7b5b32c6", [:mix], [], "hexpm", "b8b0dd77d60373e77a3d7e8afa598f325e49e8663a51bcc2b88ef41838cca755"}, "decimal": {:hex, :decimal, "2.1.1", "5611dca5d4b2c3dd497dec8f68751f1f1a54755e8ed2a966c2633cf885973ad6", [:mix], [], "hexpm", "53cfe5f497ed0e7771ae1a475575603d77425099ba5faef9394932b35020ffcc"},
"earmark": {:hex, :earmark, "1.4.39", "acdb2f02c536471029dbcc509fbd6b94b89f40ad7729fb3f68f4b6944843f01d", [:mix], [{:earmark_parser, "~> 1.4.33", [hex: :earmark_parser, repo: "hexpm", optional: false]}], "hexpm", "156c9d8ec3cbeccdbf26216d8247bdeeacc8c76b4d9eee7554be2f1b623ea440"},
"earmark_parser": {:hex, :earmark_parser, "1.4.33", "3c3fd9673bb5dcc9edc28dd90f50c87ce506d1f71b70e3de69aa8154bc695d44", [:mix], [], "hexpm", "2d526833729b59b9fdb85785078697c72ac5e5066350663e5be6a1182da61b8f"},
"ecto": {:hex, :ecto, "3.10.3", "eb2ae2eecd210b4eb8bece1217b297ad4ff824b4384c0e3fdd28aaf96edd6135", [:mix], [{:decimal, "~> 1.6 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "44bec74e2364d491d70f7e42cd0d690922659d329f6465e89feb8a34e8cd3433"},
"ecto_sql": {:hex, :ecto_sql, "3.10.1", "6ea6b3036a0b0ca94c2a02613fd9f742614b5cfe494c41af2e6571bb034dd94c", [:mix], [{:db_connection, "~> 2.4.1 or ~> 2.5", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.10.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.6.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.16.0 or ~> 0.17.0 or ~> 1.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:tds, "~> 2.1.1 or ~> 2.2", [hex: :tds, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "f6a25bdbbd695f12c8171eaff0851fa4c8e72eec1e98c7364402dda9ce11c56b"},
"enough": {:hex, :enough, "0.1.0", "0254710c52d324e2dadde54cb56fbb80a792c2eb285669b8379efd0752bf89f0", [:rebar3], [], "hexpm", "0460c7abda5f5e0ea592b12bc6976b8a5c4b96e42f332059cd396525374bf9a1"}, "enough": {:hex, :enough, "0.1.0", "0254710c52d324e2dadde54cb56fbb80a792c2eb285669b8379efd0752bf89f0", [:rebar3], [], "hexpm", "0460c7abda5f5e0ea592b12bc6976b8a5c4b96e42f332059cd396525374bf9a1"},
"jason": {:hex, :jason, "1.4.0", "e855647bc964a44e2f67df589ccf49105ae039d4179db7f6271dfd3843dc27e6", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "79a3791085b2a0f743ca04cec0f7be26443738779d09302e01318f97bdb82121"}, "jason": {:hex, :jason, "1.4.1", "af1504e35f629ddcdd6addb3513c3853991f694921b1b9368b0bd32beb9f1b63", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "fbb01ecdfd565b56261302f7e1fcc27c4fb8f32d56eab74db621fc154604a7a1"},
"mime": {:hex, :mime, "2.0.3", "3676436d3d1f7b81b5a2d2bd8405f412c677558c81b1c92be58c00562bb59095", [:mix], [], "hexpm", "27a30bf0db44d25eecba73755acf4068cbfe26a4372f9eb3e4ea3a45956bff6b"}, "mime": {:hex, :mime, "2.0.5", "dc34c8efd439abe6ae0343edbb8556f4d63f178594894720607772a041b04b02", [:mix], [], "hexpm", "da0d64a365c45bc9935cc5c8a7fc5e49a0e0f9932a761c55d6c52b142780a05c"},
"plug": {:hex, :plug, "1.14.2", "cff7d4ec45b4ae176a227acd94a7ab536d9b37b942c8e8fa6dfc0fff98ff4d80", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "842fc50187e13cf4ac3b253d47d9474ed6c296a8732752835ce4a86acdf68d13"}, "plug": {:hex, :plug, "1.14.2", "cff7d4ec45b4ae176a227acd94a7ab536d9b37b942c8e8fa6dfc0fff98ff4d80", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "842fc50187e13cf4ac3b253d47d9474ed6c296a8732752835ce4a86acdf68d13"},
"plug_cowboy": {:hex, :plug_cowboy, "2.6.1", "9a3bbfceeb65eff5f39dab529e5cd79137ac36e913c02067dba3963a26efe9b2", [:mix], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:cowboy_telemetry, "~> 0.3", [hex: :cowboy_telemetry, repo: "hexpm", optional: false]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "de36e1a21f451a18b790f37765db198075c25875c64834bcc82d90b309eb6613"}, "plug_cowboy": {:hex, :plug_cowboy, "2.6.1", "9a3bbfceeb65eff5f39dab529e5cd79137ac36e913c02067dba3963a26efe9b2", [:mix], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:cowboy_telemetry, "~> 0.3", [hex: :cowboy_telemetry, repo: "hexpm", optional: false]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "de36e1a21f451a18b790f37765db198075c25875c64834bcc82d90b309eb6613"},
"plug_crypto": {:hex, :plug_crypto, "1.2.5", "918772575e48e81e455818229bf719d4ab4181fcbf7f85b68a35620f78d89ced", [:mix], [], "hexpm", "26549a1d6345e2172eb1c233866756ae44a9609bd33ee6f99147ab3fd87fd842"}, "plug_crypto": {:hex, :plug_crypto, "1.2.5", "918772575e48e81e455818229bf719d4ab4181fcbf7f85b68a35620f78d89ced", [:mix], [], "hexpm", "26549a1d6345e2172eb1c233866756ae44a9609bd33ee6f99147ab3fd87fd842"},
"postgrex": {:hex, :postgrex, "0.17.2", "a3ec9e3239d9b33f1e5841565c4eb200055c52cc0757a22b63ca2d529bbe764c", [:mix], [{:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "80a918a9e9531d39f7bd70621422f3ebc93c01618c645f2d91306f50041ed90c"},
"ranch": {:hex, :ranch, "1.8.0", "8c7a100a139fd57f17327b6413e4167ac559fbc04ca7448e9be9057311597a1d", [:make, :rebar3], [], "hexpm", "49fbcfd3682fab1f5d109351b61257676da1a2fdbe295904176d5e521a2ddfe5"}, "ranch": {:hex, :ranch, "1.8.0", "8c7a100a139fd57f17327b6413e4167ac559fbc04ca7448e9be9057311597a1d", [:make, :rebar3], [], "hexpm", "49fbcfd3682fab1f5d109351b61257676da1a2fdbe295904176d5e521a2ddfe5"},
"systemd": {:hex, :systemd, "0.6.2", "aaa24f1e3e6cb978c45369768b1abd766a0dbff637ed61254ca64797bcec9963", [:rebar3], [{:enough, "~> 0.1.0", [hex: :enough, repo: "hexpm", optional: false]}], "hexpm", "5062b911800c1ab05157c7bf9a9fbe23dd24c58891c87fd12d2e3ed8fc1708b8"}, "systemd": {:hex, :systemd, "0.6.2", "aaa24f1e3e6cb978c45369768b1abd766a0dbff637ed61254ca64797bcec9963", [:rebar3], [{:enough, "~> 0.1.0", [hex: :enough, repo: "hexpm", optional: false]}], "hexpm", "5062b911800c1ab05157c7bf9a9fbe23dd24c58891c87fd12d2e3ed8fc1708b8"},
"telemetry": {:hex, :telemetry, "1.2.1", "68fdfe8d8f05a8428483a97d7aab2f268aaff24b49e0f599faa091f1d4e7f61c", [:rebar3], [], "hexpm", "dad9ce9d8effc621708f99eac538ef1cbe05d6a874dd741de2e689c47feafed5"}, "telemetry": {:hex, :telemetry, "1.2.1", "68fdfe8d8f05a8428483a97d7aab2f268aaff24b49e0f599faa091f1d4e7f61c", [:rebar3], [], "hexpm", "dad9ce9d8effc621708f99eac538ef1cbe05d6a874dd741de2e689c47feafed5"},

View file

@ -190,5 +190,6 @@ let
beamDeps = [ ]; beamDeps = [ ];
}; };
}; };
in self in
self

View file

@ -0,0 +1,15 @@
defmodule BdfrBrowser.Repo.Migrations.CreatePosts do
use Ecto.Migration
def change do
create table(:posts) do
add :title, :string, size: 1024
add :selftext, :text
add :url, :string, size: 2048
add :permalink, :string, size: 2048
add :author, :string
add :upvote_ratio, :float
add :posted_at, :utc_datetime
end
end
end

View file

@ -0,0 +1,17 @@
defmodule BdfrBrowser.Repo.Migrations.CreateSubreddits do
use Ecto.Migration
def change do
create table(:subreddits, primary_key: false) do
add :id, :bigserial, primary_key: true
add :name, :string, size: 1024, unique: true
end
alter table("posts") do
add :subreddit_id, references(:subreddits, type: :bigserial)
end
create index("posts", :subreddit_id)
create index("subreddits", :name, unique: true)
end
end

View file

@ -0,0 +1,17 @@
defmodule BdfrBrowser.Repo.Migrations.CreateComments do
use Ecto.Migration
def change do
create table(:comments) do
add :author, :string
add :body, :text
add :score, :integer
add :posted_at, :utc_datetime
add :post_id, references(:posts)
add :parent_id, references(:comments)
end
create index("comments", :post_id)
create index("comments", :parent_id)
end
end

View file

@ -0,0 +1,9 @@
defmodule BdfrBrowser.Repo.Migrations.AddPostFilename do
use Ecto.Migration
def change do
alter table("posts") do
add :filename, :string, size: 2048
end
end
end

View file

@ -10,7 +10,7 @@
<footer class="blockquote-footer"> <footer class="blockquote-footer">
<%= comment.author %>, <%= comment.author %>,
<small><%= trunc(comment.created_utc) |> DateTime.from_unix!() |> DateTime.to_iso8601() %></small> <small><%= DateTime.to_iso8601(comment.posted_at) %></small>
</footer> </footer>
</blockquote> </blockquote>
</div> </div>
@ -18,6 +18,6 @@
<br> <br>
</div> </div>
<%= for reply <- comment.replies do %> <%= for reply <- BdfrBrowser.Repo.preload(comment, :children).children do %>
<%= EEx.eval_file(comment_template, comment: reply, level: level + 1, comment_template: comment_template) %> <%= EEx.eval_file(comment_template, comment: reply, level: level + 1, comment_template: comment_template) %>
<% end %> <% end %>

View file

@ -17,7 +17,7 @@
<p> <p>
<small><a href="https://reddit.com/user/<%= post.author %>"><%= post.author %></a></small> <small><a href="https://reddit.com/user/<%= post.author %>"><%= post.author %></a></small>
- -
<small><%= trunc(post.created_utc) |> DateTime.from_unix!() |> DateTime.to_iso8601() %></small> <small><%= DateTime.to_iso8601(post.posted_at) %></small>
- -
<a href="https://reddit.com<%= post.permalink %>">Open reddit</a> <a href="https://reddit.com<%= post.permalink %>">Open reddit</a>
</p> </p>

View file

@ -19,9 +19,9 @@
<%= for post <- posts do %> <%= for post <- posts do %>
<div class="card"> <div class="card">
<div class="card-body"> <div class="card-body">
<h5 class="card-title"><a href="/r/<%= subreddit %>/<%= date %>/<%= URI.encode(post.filename, &URI.char_unreserved?/1) %>"><%= post.title %></a></h5> <h5 class="card-title"><a href="/r/<%= subreddit %>/<%= date %>/<%= post.id %>"><%= post.title %></a></h5>
<h6 class="card-subtitle mb-2 text-body-secondary"> <h6 class="card-subtitle mb-2 text-body-secondary">
<%= post.num_comments %> comment(s) - <%= trunc(post.created_utc) |> DateTime.from_unix!() |> DateTime.to_iso8601() %> <%= post.num_comments %> comment(s) - <%= DateTime.to_iso8601(post.posted_at) %>
</h6> </h6>
</div> </div>
</div> </div>