diff --git a/.formatter.exs b/.formatter.exs index b717f1d..433fea5 100644 --- a/.formatter.exs +++ b/.formatter.exs @@ -4,5 +4,5 @@ "{config,lib,test}/**/*.{ex,exs}" ], line_length: 120, - import_deps: [:plug] + import_deps: [:ecto, :ecto_sql, :plug] ] diff --git a/.gitignore b/.gitignore index ce63943..5cd453b 100644 --- a/.gitignore +++ b/.gitignore @@ -19,6 +19,7 @@ erl_crash.dump # nix /.direnv /.elixir_ls +.pre-commit-config.yaml # Exclude releases /rel/* diff --git a/config/config.exs b/config/config.exs index 5bcfdcc..fa494ac 100644 --- a/config/config.exs +++ b/config/config.exs @@ -1,7 +1,16 @@ import Config +config :bdfr_browser, + ecto_repos: [BdfrBrowser.Repo] + +config :bdfr_browser, BdfrBrowser.Repo, + migration_primary_key: [name: :id, type: :string], + migration_foreign_key: [column: :id, type: :string] + config :logger, backends: [], - level: :warning, + level: :info, handle_otp_reports: false, handle_sasl_reports: false + +import_config "#{Mix.env()}.exs" diff --git a/config/dev.exs b/config/dev.exs new file mode 100644 index 0000000..f17fe74 --- /dev/null +++ b/config/dev.exs @@ -0,0 +1,5 @@ +import Config + +config :logger, + backends: [:console], + level: :debug diff --git a/config/prod.exs b/config/prod.exs new file mode 100644 index 0000000..becde76 --- /dev/null +++ b/config/prod.exs @@ -0,0 +1 @@ +import Config diff --git a/config/runtime.exs b/config/runtime.exs index 82a7c58..8994f86 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -4,3 +4,9 @@ config :bdfr_browser, base_directory: System.get_env("BDFR_BROWSER_BASE_DIRECTORY", "/nonexistant"), http_ip: to_charlist(System.get_env("BDFR_BROWSER_HTTP_IP", "127.0.0.1")), http_port: String.to_integer(System.get_env("BDFR_BROWSER_HTTP_PORT", "4040")) + +config :bdfr_browser, BdfrBrowser.Repo, + database: System.get_env("BDFR_BROWSER_REPO_DATABASE", "postgres"), + username: System.get_env("BDFR_BROWSER_REPO_USER", "bdfr-browser"), + password: System.get_env("BDFR_BROWSER_REPO_PASSWORD", ""), + hostname: System.get_env("BDFR_BROWSER_REPO_HOST", "localhost") diff --git a/config/test.exs b/config/test.exs new file mode 100644 index 0000000..becde76 --- /dev/null +++ b/config/test.exs @@ -0,0 +1 @@ +import Config diff --git a/flake.lock b/flake.lock index b8fa6a9..87006d9 100644 --- a/flake.lock +++ b/flake.lock @@ -1,15 +1,31 @@ { "nodes": { + "flake-compat": { + "flake": false, + "locked": { + "lastModified": 1673956053, + "narHash": "sha256-4gtG9iQuiKITOjNQQeQIpoIB6b16fm+504Ch3sNKLd8=", + "owner": "edolstra", + "repo": "flake-compat", + "rev": "35bb57c0c8d8b62bbfd284272c928ceb64ddbde9", + "type": "github" + }, + "original": { + "owner": "edolstra", + "repo": "flake-compat", + "type": "github" + } + }, "flake-parts": { "inputs": { "nixpkgs-lib": "nixpkgs-lib" }, "locked": { - "lastModified": 1683560683, - "narHash": "sha256-XAygPMN5Xnk/W2c1aW0jyEa6lfMDZWlQgiNtmHXytPc=", + "lastModified": 1690933134, + "narHash": "sha256-ab989mN63fQZBFrkk4Q8bYxQCktuHmBIBqUG1jl6/FQ=", "owner": "hercules-ci", "repo": "flake-parts", - "rev": "006c75898cf814ef9497252b022e91c946ba8e17", + "rev": "59cf3f1447cfc75087e7273b04b31e689a8599fb", "type": "github" }, "original": { @@ -18,13 +34,66 @@ "type": "github" } }, + "flake-root": { + "locked": { + "lastModified": 1680964220, + "narHash": "sha256-dIdTYcf+KW9a4pKHsEbddvLVSfR1yiAJynzg2x0nfWg=", + "owner": "srid", + "repo": "flake-root", + "rev": "f1c0b93d05bdbea6c011136ba1a135c80c5b326c", + "type": "github" + }, + "original": { + "owner": "srid", + "repo": "flake-root", + "type": "github" + } + }, + "flake-utils": { + "inputs": { + "systems": "systems" + }, + "locked": { + "lastModified": 1685518550, + "narHash": "sha256-o2d0KcvaXzTrPRIo0kOLV0/QXHhDQ5DTi+OxcjO8xqY=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "a1720a10a6cfe8234c0e93907ffe81be440f4cef", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "gitignore": { + "inputs": { + "nixpkgs": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1660459072, + "narHash": "sha256-8DFJjXG8zqoONA1vXtgeKXy68KdJL5UaXR8NtVMUbx8=", + "owner": "hercules-ci", + "repo": "gitignore.nix", + "rev": "a20de23b925fd8264fd7fad6454652e142fd7f73", + "type": "github" + }, + "original": { + "owner": "hercules-ci", + "repo": "gitignore.nix", + "type": "github" + } + }, "nixpkgs": { "locked": { - "lastModified": 1684242266, - "narHash": "sha256-uaCQ2k1bmojHKjWQngvnnnxQJMY8zi1zq527HdWgQf8=", + "lastModified": 1691853136, + "narHash": "sha256-wTzDsRV4HN8A2Sl0SVQY0q8ILs90CD43Ha//7gNZE+E=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "7e0743a5aea1dc755d4b761daf75b20aa486fdad", + "rev": "f0451844bbdf545f696f029d1448de4906c7f753", "type": "github" }, "original": { @@ -37,11 +106,11 @@ "nixpkgs-lib": { "locked": { "dir": "lib", - "lastModified": 1682879489, - "narHash": "sha256-sASwo8gBt7JDnOOstnps90K1wxmVfyhsTPPNTGBPjjg=", + "lastModified": 1690881714, + "narHash": "sha256-h/nXluEqdiQHs1oSgkOOWF+j8gcJMWhwnZ9PFabN6q0=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "da45bf6ec7bbcc5d1e14d3795c025199f28e0de0", + "rev": "9e1960bc196baf6881340d53dccb203a951745a2", "type": "github" }, "original": { @@ -52,10 +121,107 @@ "type": "github" } }, + "nixpkgs-stable": { + "locked": { + "lastModified": 1685801374, + "narHash": "sha256-otaSUoFEMM+LjBI1XL/xGB5ao6IwnZOXc47qhIgJe8U=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "c37ca420157f4abc31e26f436c1145f8951ff373", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-23.05", + "repo": "nixpkgs", + "type": "github" + } + }, + "pre-commit-hooks-nix": { + "inputs": { + "flake-compat": "flake-compat", + "flake-utils": "flake-utils", + "gitignore": [ + "gitignore" + ], + "nixpkgs": [ + "nixpkgs" + ], + "nixpkgs-stable": "nixpkgs-stable" + }, + "locked": { + "lastModified": 1691747570, + "narHash": "sha256-J3fnIwJtHVQ0tK2JMBv4oAmII+1mCdXdpeCxtIsrL2A=", + "owner": "cachix", + "repo": "pre-commit-hooks.nix", + "rev": "c5ac3aa3324bd8aebe8622a3fc92eeb3975d317a", + "type": "github" + }, + "original": { + "owner": "cachix", + "repo": "pre-commit-hooks.nix", + "type": "github" + } + }, + "process-compose-flake": { + "locked": { + "lastModified": 1691182187, + "narHash": "sha256-dmHKgFXstdfX7nfnpbzR5H4DWGdWo610zsW9BCtI2WQ=", + "owner": "Platonic-Systems", + "repo": "process-compose-flake", + "rev": "bacdaf54ffe3a2c1734fd973a95e6b39b1560c2e", + "type": "github" + }, + "original": { + "owner": "Platonic-Systems", + "repo": "process-compose-flake", + "type": "github" + } + }, "root": { "inputs": { "flake-parts": "flake-parts", - "nixpkgs": "nixpkgs" + "flake-root": "flake-root", + "gitignore": "gitignore", + "nixpkgs": "nixpkgs", + "pre-commit-hooks-nix": "pre-commit-hooks-nix", + "process-compose-flake": "process-compose-flake", + "treefmt-nix": "treefmt-nix" + } + }, + "systems": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + }, + "treefmt-nix": { + "inputs": { + "nixpkgs": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1691833704, + "narHash": "sha256-ASGhgGduEgcD3gQZhGr8xtmZ3PlVY+m2HuPnIZDbu78=", + "owner": "numtide", + "repo": "treefmt-nix", + "rev": "19dee4bf6001849006a63f3435247316b0488e99", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "treefmt-nix", + "type": "github" } } }, diff --git a/flake.nix b/flake.nix index bff589b..0476072 100644 --- a/flake.nix +++ b/flake.nix @@ -1,47 +1,193 @@ { description = "bdfr-browser development environment"; + inputs = { nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable"; + + # Tools + flake-parts.url = "github:hercules-ci/flake-parts"; + + flake-root.url = "github:srid/flake-root"; + + treefmt-nix = { + url = "github:numtide/treefmt-nix"; + inputs.nixpkgs.follows = "nixpkgs"; + }; + + gitignore = { + url = "github:hercules-ci/gitignore.nix"; + inputs.nixpkgs.follows = "nixpkgs"; + }; + + pre-commit-hooks-nix = { + url = "github:cachix/pre-commit-hooks.nix"; + inputs.nixpkgs.follows = "nixpkgs"; + inputs.gitignore.follows = "gitignore"; + }; + + process-compose-flake.url = "github:Platonic-Systems/process-compose-flake"; }; - outputs = inputs@{ flake-parts, ... }: + outputs = inputs@{ flake-parts, gitignore, ... }: flake-parts.lib.mkFlake { inherit inputs; } { systems = [ "aarch64-darwin" "x86_64-linux" "aarch64-linux" ]; - perSystem = { pkgs, lib, self', ... }: + imports = [ + inputs.flake-root.flakeModule + inputs.treefmt-nix.flakeModule + inputs.pre-commit-hooks-nix.flakeModule + inputs.process-compose-flake.flakeModule + ]; + + perSystem = { pkgs, lib, config, self', ... }: let pname = "bdfr-browser"; version = "0.0.1"; - erlang = pkgs.beam.interpreters.erlangR25; - beamPackages = pkgs.beam.packagesWith erlang; - elixir = beamPackages.elixir_1_14; + erlang = pkgs.beam.interpreters.erlangR26; + beamPackagesPrev = pkgs.beam.packagesWith erlang; + elixir = beamPackagesPrev.elixir_1_15; + + beamPackages = beamPackagesPrev // rec { + inherit erlang elixir; + hex = beamPackagesPrev.hex.override { inherit elixir; }; + buildMix = beamPackagesPrev.buildMix.override { inherit elixir erlang hex; }; + mixRelease = beamPackagesPrev.mixRelease.override { inherit erlang elixir; }; + }; + + postgres = pkgs.postgresql_15; inherit (pkgs.stdenv) isDarwin; + inherit (gitignore.lib) gitignoreSource; in { + treefmt = { + inherit (config.flake-root) projectRootFile; + flakeCheck = false; + + programs = { + mix-format = { + enable = true; + package = elixir; + }; + + nixpkgs-fmt.enable = true; + }; + }; + + pre-commit = { + check.enable = false; + + settings = { + excludes = [ "mix.nix" ]; + + hooks = { + deadnix.enable = true; + statix.enable = true; + treefmt.enable = true; + }; + }; + }; + devShells.default = pkgs.mkShell { - packages = (with pkgs; [ + name = pname; + + nativeBuildInputs = [ erlang elixir + postgres + ] ++ lib.optionals isDarwin (with pkgs.darwin.apple_sdk.frameworks; + [ + CoreFoundation + CoreServices + ]); - beamPackages.elixir-ls - mix2nix - ]) ++ lib.optionals isDarwin (with pkgs.darwin.apple_sdk.frameworks; [ - CoreFoundation - CoreServices - ]); + packages = [ + pkgs.mix2nix + self'.packages.bdfr-browser-dev + ]; + + inputsFrom = [ + config.flake-root.devShell + config.treefmt.build.devShell + config.pre-commit.devShell + ]; ERL_INCLUDE_PATH = "${erlang}/lib/erlang/usr/include"; + TREEFMT_CONFIG_FILE = config.treefmt.build.configFile; }; packages.default = beamPackages.mixRelease { inherit pname version; - src = ./.; + src = gitignoreSource ./.; mixNixDeps = import ./mix.nix { inherit lib beamPackages; }; }; + + process-compose."${pname}-dev" = + let + db-host = "127.0.0.1"; + db-user = "bdfr-browser"; + in + { + port = 18808; + + settings = { + environment = { + BDFR_BROWSER_BASE_DIRECTORY = "/Volumes/MediaScraper/Reddit"; + BDFR_BROWSER_REPO_USER = db-user; + BDFR_BROWSER_REPO_HOST = db-host; + }; + + processes = { + db-init.command = '' + if [ ! -d "$PWD/.direnv/postgres/data" ]; then + echo "Initializing database ..." + mkdir -p "$PWD/.direnv/postgres" + ${postgres}/bin/initdb --username ${db-user} --pgdata "$PWD/.direnv/postgres/data" --auth trust + else + echo "Database already initialized" + fi + ''; + + db = { + command = "${postgres}/bin/postgres -D $PWD/.direnv/postgres/data"; + + depends_on.db-init.condition = "process_completed_successfully"; + + readiness_probe.exec.command = "PGCONNECT_TIMEOUT=1 ${postgres}/bin/psql -h ${db-host} -U ${db-user} -l"; + }; + + app-setup.command = '' + mix local.hex --if-missing --force + mix local.rebar --force + mix deps.get + ''; + + app-compile = { + command = "mix release --overwrite"; + + depends_on.app-setup.condition = "process_completed_successfully"; + }; + + app = { + command = "$PWD/_build/dev/rel/bdfr_browser/bin/bdfr_browser start"; + + depends_on = { + db.condition = "process_healthy"; + app-compile.condition = "process_completed_successfully"; + }; + + readiness_probe.http_get = { + host = "127.0.0.1"; + port = 4040; + path = "/_ping"; + }; + }; + }; + }; + }; }; }; } diff --git a/lib/bdfr_browser/application.ex b/lib/bdfr_browser/application.ex index 7e20288..cc16286 100644 --- a/lib/bdfr_browser/application.ex +++ b/lib/bdfr_browser/application.ex @@ -7,10 +7,14 @@ defmodule BdfrBrowser.Application do @impl true def start(_type, _args) do + repos = Application.fetch_env!(:bdfr_browser, :ecto_repos) {:ok, http_ip} = :inet.parse_address(Application.fetch_env!(:bdfr_browser, :http_ip)) http_port = Application.fetch_env!(:bdfr_browser, :http_port) children = [ + {Ecto.Migrator, repos: repos, skip: System.get_env("SKIP_MIGRATIONS") == "true"}, + BdfrBrowser.Repo, + BdfrBrowser.Importer, {Plug.Cowboy, scheme: :http, plug: BdfrBrowser.HTTP.Plug, options: [ip: http_ip, port: http_port]}, :systemd.ready() ] diff --git a/lib/bdfr_browser/comment.ex b/lib/bdfr_browser/comment.ex new file mode 100644 index 0000000..437da46 --- /dev/null +++ b/lib/bdfr_browser/comment.ex @@ -0,0 +1,19 @@ +defmodule BdfrBrowser.Comment do + use Ecto.Schema + + alias BdfrBrowser.Post + + @primary_key {:id, :string, autogenerate: false} + @foreign_key_type :string + + schema "comments" do + field :author, :string + field :body, :string + field :score, :integer + field :posted_at, :utc_datetime + + belongs_to :post, Post + belongs_to :parent, __MODULE__ + has_many :children, __MODULE__, foreign_key: :parent_id + end +end diff --git a/lib/bdfr_browser/http/plug.ex b/lib/bdfr_browser/http/plug.ex index ab79b7f..f599c28 100644 --- a/lib/bdfr_browser/http/plug.ex +++ b/lib/bdfr_browser/http/plug.ex @@ -1,12 +1,14 @@ defmodule BdfrBrowser.HTTP.Plug do use Plug.Router + alias BdfrBrowser.{Repo, Post, Subreddit} + plug :match plug :dispatch get "/" do tpl_params = [ - subreddits: list_folders(sort: :asc) + subreddits: Subreddit.names() |> Repo.all() ] tpl_file = Application.app_dir(:bdfr_browser, "priv/templates/http/index.eex") @@ -18,9 +20,11 @@ defmodule BdfrBrowser.HTTP.Plug do end get "/r/:subreddit" do + subreddit_record = Repo.get_by(Subreddit, name: subreddit) + tpl_params = [ subreddit: subreddit, - dates: list_folders(paths: [subreddit]) + dates: subreddit_record |> Post.date_listing() |> Repo.all() ] tpl_file = Application.app_dir(:bdfr_browser, "priv/templates/http/subreddit.eex") @@ -32,10 +36,12 @@ defmodule BdfrBrowser.HTTP.Plug do end get "/r/:subreddit/:date" do + subreddit_record = Repo.get_by(Subreddit, name: subreddit) + tpl_params = [ subreddit: subreddit, date: date, - posts: read_posts(paths: [subreddit, date], ext: ".json") + posts: subreddit_record |> Post.during_month(date) |> Repo.all() ] tpl_file = Application.app_dir(:bdfr_browser, "priv/templates/http/subreddit_posts.eex") @@ -46,12 +52,14 @@ defmodule BdfrBrowser.HTTP.Plug do |> send_resp(200, content) end - get "/r/:subreddit/:date/:post" do + get "/r/:subreddit/:date/:id" do + post_record = Post |> Repo.get(id) |> Repo.preload(comments: :children) + tpl_params = [ subreddit: subreddit, date: date, - post: read_post(post, paths: [subreddit, date]), - media: post_media(post, paths: [subreddit, date]), + post: post_record, + media: post_media(post_record.filename, paths: [subreddit, date]), comment_template: Application.app_dir(:bdfr_browser, "priv/templates/http/_comment.eex") ] @@ -73,66 +81,21 @@ defmodule BdfrBrowser.HTTP.Plug do |> send_resp(200, media) end + get "/_import" do + :ok = BdfrBrowser.Importer.background_import() + send_resp(conn, 200, "IMPORTING") + end + + get "/_ping" do + send_resp(conn, 200, "PONG") + end + match _ do send_resp(conn, 404, "Not Found") end # Helper - defp list_folders(args) do - paths = Keyword.get(args, :paths, []) - extname = Keyword.get(args, :ext, "") - sort = Keyword.get(args, :sort, :desc) - base_directory = Application.fetch_env!(:bdfr_browser, :base_directory) - - [base_directory | paths] - |> Path.join() - |> File.ls!() - |> Enum.filter(fn s -> not String.starts_with?(s, ".") and Path.extname(s) == extname end) - |> Enum.sort_by(&String.downcase/1, sort) - end - - defp read_posts(args) do - posts = list_folders(args) - sort = Keyword.get(args, :sort, :desc) - - base_directory = Application.fetch_env!(:bdfr_browser, :base_directory) - post_dir = Path.join([base_directory | Keyword.fetch!(args, :paths)]) - - compact_posts = - for post <- posts do - {:ok, content} = [post_dir, post] |> Path.join() |> File.read!() |> Jason.decode() - - %{ - title: extract_title(content["title"]), - author: content["author"], - num_comments: content["num_comments"], - created_utc: content["created_utc"], - filename: Path.basename(post, ".json") - } - end - - Enum.sort_by(compact_posts, fn p -> p.created_utc end, sort) - end - - defp extract_title(title) do - if String.length(title) == 0 do - "Empty Title" - else - title - end - end - - defp read_post(post, args) do - base_directory = Application.fetch_env!(:bdfr_browser, :base_directory) - post_dir = Path.join([base_directory | Keyword.fetch!(args, :paths)]) - post_file = "#{post}.json" - - {:ok, content} = [post_dir, post_file] |> Path.join() |> File.read!() |> Jason.decode(keys: :atoms) - - content - end - defp post_media(post, args) do base_directory = Application.fetch_env!(:bdfr_browser, :base_directory) post_dir = Path.join([base_directory | Keyword.fetch!(args, :paths)]) diff --git a/lib/bdfr_browser/importer.ex b/lib/bdfr_browser/importer.ex new file mode 100644 index 0000000..4b47401 --- /dev/null +++ b/lib/bdfr_browser/importer.ex @@ -0,0 +1,145 @@ +defmodule BdfrBrowser.Importer do + require Logger + + use GenServer + + alias BdfrBrowser.{Comment, Post, Repo, Subreddit} + + def start_link([]) do + GenServer.start_link(__MODULE__, [], name: __MODULE__) + end + + def subreddits do + _ = Logger.info("Importing subreddits ...") + + folders = list_folders(sort: :asc) + + for folder <- folders do + %Subreddit{name: folder} + |> Repo.insert( + on_conflict: :nothing, + conflict_target: :name + ) + end + end + + def posts_and_comments do + _ = Logger.info("Importing posts and comments ...") + + result = + for subreddit <- list_folders(sort: :asc) do + _ = Logger.info("Importing entries from `#{subreddit}' ...") + + subreddit_record = Repo.get_by(Subreddit, name: subreddit) + + for date <- list_folders(paths: [subreddit]) do + _ = Logger.debug("Importing entries from `#{subreddit}' on `#{date}' ...") + + for post <- read_posts(paths: [subreddit, date], ext: ".json") do + _ = Logger.debug("Importing `#{post["id"]}' from `#{subreddit}' ...") + + {:ok, post_record} = import_post(post, subreddit_record) + comment_records = for comment <- post["comments"], do: import_comment(comment, post_record, nil) + + {post_record, List.flatten(comment_records)} + end + end + end + + List.flatten(result) + end + + def background_import do + GenServer.cast(__MODULE__, :background_import) + end + + # Callbacks + + @impl true + def init([]) do + {:ok, nil} + end + + @impl true + def handle_cast(:background_import, state) do + _ = subreddits() + _ = posts_and_comments() + {:noreply, state} + end + + # Helper + + defp list_folders(args) do + paths = Keyword.get(args, :paths, []) + extname = Keyword.get(args, :ext, "") + sort = Keyword.get(args, :sort, :desc) + base_directory = Application.fetch_env!(:bdfr_browser, :base_directory) + + [base_directory | paths] + |> Path.join() + |> File.ls!() + |> Enum.filter(fn s -> not String.starts_with?(s, ".") and Path.extname(s) == extname end) + |> Enum.sort_by(&String.downcase/1, sort) + end + + defp read_posts(args) do + posts = list_folders(args) + sort = Keyword.get(args, :sort, :desc) + + base_directory = Application.fetch_env!(:bdfr_browser, :base_directory) + post_dir = Path.join([base_directory | Keyword.fetch!(args, :paths)]) + + parsed_posts = + for post <- posts do + file_path = Path.join([post_dir, post]) + parsed = file_path |> File.read!() |> Jason.decode!() + Map.put(parsed, "filename", post) + end + + Enum.sort_by(parsed_posts, fn p -> p["created_utc"] end, sort) + end + + defp import_post(post, subreddit) do + id = post["id"] + + %Post{ + id: id, + title: post["title"], + selftext: post["selftext"], + url: post["url"], + permalink: post["permalink"], + author: post["author"], + upvote_ratio: post["upvote_ratio"], + posted_at: DateTime.from_unix!(trunc(post["created_utc"])), + filename: Path.basename(post["filename"], ".json"), + subreddit: subreddit + } + |> Repo.insert( + on_conflict: [set: [id: id]], + conflict_target: :id + ) + end + + defp import_comment(comment, post, parent) do + id = comment["id"] + + {:ok, parent} = + %Comment{ + id: id, + author: comment["author"], + body: comment["body"], + score: comment["score"], + posted_at: DateTime.from_unix!(trunc(comment["created_utc"])), + post: post, + parent: parent + } + |> Repo.insert( + on_conflict: [set: [id: id]], + conflict_target: :id + ) + + children = for child <- comment["replies"], do: import_comment(child, post, parent) + + [parent] ++ children + end +end diff --git a/lib/bdfr_browser/post.ex b/lib/bdfr_browser/post.ex new file mode 100644 index 0000000..26a6077 --- /dev/null +++ b/lib/bdfr_browser/post.ex @@ -0,0 +1,49 @@ +defmodule BdfrBrowser.Post do + use Ecto.Schema + + import Ecto.Query, only: [from: 2] + + alias BdfrBrowser.{Comment, Subreddit} + + @primary_key {:id, :string, autogenerate: false} + + schema "posts" do + field :title, :string + field :selftext, :string + field :url, :string + field :permalink, :string + field :author, :string + field :upvote_ratio, :float + field :posted_at, :utc_datetime + field :filename, :string + + belongs_to :subreddit, Subreddit + has_many :comments, Comment + end + + def date_listing(subreddit) do + from(p in __MODULE__, + select: fragment("to_char(?, 'YYYY-MM')", p.posted_at), + where: p.subreddit_id == ^subreddit.id, + distinct: true, + order_by: [desc: fragment("to_char(?, 'YYYY-MM')", p.posted_at)] + ) + end + + def during_month(subreddit, month_str) do + {:ok, d} = Date.from_iso8601("#{month_str}-01") + during_range(subreddit, Date.beginning_of_month(d), Date.end_of_month(d)) + end + + def during_range(subreddit, start_date, end_date) do + from(p in __MODULE__, + left_join: c in assoc(p, :comments), + select: %{id: p.id, title: p.title, author: p.author, posted_at: p.posted_at, num_comments: count(c.id)}, + where: + p.subreddit_id == ^subreddit.id and type(p.posted_at, :date) >= ^start_date and + type(p.posted_at, :date) <= ^end_date, + order_by: [desc: p.posted_at], + group_by: p.id + ) + end +end diff --git a/lib/bdfr_browser/repo.ex b/lib/bdfr_browser/repo.ex new file mode 100644 index 0000000..cea39f5 --- /dev/null +++ b/lib/bdfr_browser/repo.ex @@ -0,0 +1,5 @@ +defmodule BdfrBrowser.Repo do + use Ecto.Repo, + otp_app: :bdfr_browser, + adapter: Ecto.Adapters.Postgres +end diff --git a/lib/bdfr_browser/subreddit.ex b/lib/bdfr_browser/subreddit.ex new file mode 100644 index 0000000..e8101d9 --- /dev/null +++ b/lib/bdfr_browser/subreddit.ex @@ -0,0 +1,17 @@ +defmodule BdfrBrowser.Subreddit do + use Ecto.Schema + + import Ecto.Query, only: [from: 2] + + alias BdfrBrowser.Post + + schema "subreddits" do + field(:name, :string) + + has_many(:posts, Post) + end + + def names do + from(s in __MODULE__, select: s.name, order_by: [asc: s.name]) + end +end diff --git a/mix.exs b/mix.exs index a328b08..d1b25c4 100644 --- a/mix.exs +++ b/mix.exs @@ -21,6 +21,8 @@ defmodule BdfrBrowser.MixProject do defp deps do [ {:plug_cowboy, "~> 2.6"}, + {:ecto_sql, "~> 3.10"}, + {:postgrex, "~> 0.17"}, {:jason, "~> 1.4"}, {:earmark, "~> 1.4"}, {:systemd, "~> 0.6"} diff --git a/mix.lock b/mix.lock index 9376f51..658afa0 100644 --- a/mix.lock +++ b/mix.lock @@ -2,14 +2,19 @@ "cowboy": {:hex, :cowboy, "2.10.0", "ff9ffeff91dae4ae270dd975642997afe2a1179d94b1887863e43f681a203e26", [:make, :rebar3], [{:cowlib, "2.12.1", [hex: :cowlib, repo: "hexpm", optional: false]}, {:ranch, "1.8.0", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "3afdccb7183cc6f143cb14d3cf51fa00e53db9ec80cdcd525482f5e99bc41d6b"}, "cowboy_telemetry": {:hex, :cowboy_telemetry, "0.4.0", "f239f68b588efa7707abce16a84d0d2acf3a0f50571f8bb7f56a15865aae820c", [:rebar3], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "7d98bac1ee4565d31b62d59f8823dfd8356a169e7fcbb83831b8a5397404c9de"}, "cowlib": {:hex, :cowlib, "2.12.1", "a9fa9a625f1d2025fe6b462cb865881329b5caff8f1854d1cbc9f9533f00e1e1", [:make, :rebar3], [], "hexpm", "163b73f6367a7341b33c794c4e88e7dbfe6498ac42dcd69ef44c5bc5507c8db0"}, - "earmark": {:hex, :earmark, "1.4.38", "ba8fda946c259c6e8f6759d3647d448e9216e2c0afed8c6ae7f8ce1f7072a497", [:mix], [{:earmark_parser, "~> 1.4.32", [hex: :earmark_parser, repo: "hexpm", optional: false]}], "hexpm", "f938e30de4167e7d8f3bf588b01dc041138278dda1e5a13fb9ec89b43dd5ec7f"}, - "earmark_parser": {:hex, :earmark_parser, "1.4.32", "fa739a0ecfa34493de19426681b23f6814573faee95dfd4b4aafe15a7b5b32c6", [:mix], [], "hexpm", "b8b0dd77d60373e77a3d7e8afa598f325e49e8663a51bcc2b88ef41838cca755"}, + "db_connection": {:hex, :db_connection, "2.5.0", "bb6d4f30d35ded97b29fe80d8bd6f928a1912ca1ff110831edcd238a1973652c", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "c92d5ba26cd69ead1ff7582dbb860adeedfff39774105a4f1c92cbb654b55aa2"}, + "decimal": {:hex, :decimal, "2.1.1", "5611dca5d4b2c3dd497dec8f68751f1f1a54755e8ed2a966c2633cf885973ad6", [:mix], [], "hexpm", "53cfe5f497ed0e7771ae1a475575603d77425099ba5faef9394932b35020ffcc"}, + "earmark": {:hex, :earmark, "1.4.39", "acdb2f02c536471029dbcc509fbd6b94b89f40ad7729fb3f68f4b6944843f01d", [:mix], [{:earmark_parser, "~> 1.4.33", [hex: :earmark_parser, repo: "hexpm", optional: false]}], "hexpm", "156c9d8ec3cbeccdbf26216d8247bdeeacc8c76b4d9eee7554be2f1b623ea440"}, + "earmark_parser": {:hex, :earmark_parser, "1.4.33", "3c3fd9673bb5dcc9edc28dd90f50c87ce506d1f71b70e3de69aa8154bc695d44", [:mix], [], "hexpm", "2d526833729b59b9fdb85785078697c72ac5e5066350663e5be6a1182da61b8f"}, + "ecto": {:hex, :ecto, "3.10.3", "eb2ae2eecd210b4eb8bece1217b297ad4ff824b4384c0e3fdd28aaf96edd6135", [:mix], [{:decimal, "~> 1.6 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "44bec74e2364d491d70f7e42cd0d690922659d329f6465e89feb8a34e8cd3433"}, + "ecto_sql": {:hex, :ecto_sql, "3.10.1", "6ea6b3036a0b0ca94c2a02613fd9f742614b5cfe494c41af2e6571bb034dd94c", [:mix], [{:db_connection, "~> 2.4.1 or ~> 2.5", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.10.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.6.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.16.0 or ~> 0.17.0 or ~> 1.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:tds, "~> 2.1.1 or ~> 2.2", [hex: :tds, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "f6a25bdbbd695f12c8171eaff0851fa4c8e72eec1e98c7364402dda9ce11c56b"}, "enough": {:hex, :enough, "0.1.0", "0254710c52d324e2dadde54cb56fbb80a792c2eb285669b8379efd0752bf89f0", [:rebar3], [], "hexpm", "0460c7abda5f5e0ea592b12bc6976b8a5c4b96e42f332059cd396525374bf9a1"}, - "jason": {:hex, :jason, "1.4.0", "e855647bc964a44e2f67df589ccf49105ae039d4179db7f6271dfd3843dc27e6", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "79a3791085b2a0f743ca04cec0f7be26443738779d09302e01318f97bdb82121"}, - "mime": {:hex, :mime, "2.0.3", "3676436d3d1f7b81b5a2d2bd8405f412c677558c81b1c92be58c00562bb59095", [:mix], [], "hexpm", "27a30bf0db44d25eecba73755acf4068cbfe26a4372f9eb3e4ea3a45956bff6b"}, + "jason": {:hex, :jason, "1.4.1", "af1504e35f629ddcdd6addb3513c3853991f694921b1b9368b0bd32beb9f1b63", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "fbb01ecdfd565b56261302f7e1fcc27c4fb8f32d56eab74db621fc154604a7a1"}, + "mime": {:hex, :mime, "2.0.5", "dc34c8efd439abe6ae0343edbb8556f4d63f178594894720607772a041b04b02", [:mix], [], "hexpm", "da0d64a365c45bc9935cc5c8a7fc5e49a0e0f9932a761c55d6c52b142780a05c"}, "plug": {:hex, :plug, "1.14.2", "cff7d4ec45b4ae176a227acd94a7ab536d9b37b942c8e8fa6dfc0fff98ff4d80", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "842fc50187e13cf4ac3b253d47d9474ed6c296a8732752835ce4a86acdf68d13"}, "plug_cowboy": {:hex, :plug_cowboy, "2.6.1", "9a3bbfceeb65eff5f39dab529e5cd79137ac36e913c02067dba3963a26efe9b2", [:mix], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:cowboy_telemetry, "~> 0.3", [hex: :cowboy_telemetry, repo: "hexpm", optional: false]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "de36e1a21f451a18b790f37765db198075c25875c64834bcc82d90b309eb6613"}, "plug_crypto": {:hex, :plug_crypto, "1.2.5", "918772575e48e81e455818229bf719d4ab4181fcbf7f85b68a35620f78d89ced", [:mix], [], "hexpm", "26549a1d6345e2172eb1c233866756ae44a9609bd33ee6f99147ab3fd87fd842"}, + "postgrex": {:hex, :postgrex, "0.17.2", "a3ec9e3239d9b33f1e5841565c4eb200055c52cc0757a22b63ca2d529bbe764c", [:mix], [{:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "80a918a9e9531d39f7bd70621422f3ebc93c01618c645f2d91306f50041ed90c"}, "ranch": {:hex, :ranch, "1.8.0", "8c7a100a139fd57f17327b6413e4167ac559fbc04ca7448e9be9057311597a1d", [:make, :rebar3], [], "hexpm", "49fbcfd3682fab1f5d109351b61257676da1a2fdbe295904176d5e521a2ddfe5"}, "systemd": {:hex, :systemd, "0.6.2", "aaa24f1e3e6cb978c45369768b1abd766a0dbff637ed61254ca64797bcec9963", [:rebar3], [{:enough, "~> 0.1.0", [hex: :enough, repo: "hexpm", optional: false]}], "hexpm", "5062b911800c1ab05157c7bf9a9fbe23dd24c58891c87fd12d2e3ed8fc1708b8"}, "telemetry": {:hex, :telemetry, "1.2.1", "68fdfe8d8f05a8428483a97d7aab2f268aaff24b49e0f599faa091f1d4e7f61c", [:rebar3], [], "hexpm", "dad9ce9d8effc621708f99eac538ef1cbe05d6a874dd741de2e689c47feafed5"}, diff --git a/mix.nix b/mix.nix index 421aa20..9b70dff 100644 --- a/mix.nix +++ b/mix.nix @@ -1,4 +1,4 @@ -{ lib, beamPackages, overrides ? (x: y: {}) }: +{ lib, beamPackages, overrides ? (x: y: { }) }: let buildRebar3 = lib.makeOverridable beamPackages.buildRebar3; @@ -44,7 +44,7 @@ let sha256 = "1c4dgi8canscyjgddp22mjc69znvwy44wk3r7jrl2wvs6vv76fqn"; }; - beamDeps = []; + beamDeps = [ ]; }; earmark = buildMix rec { @@ -70,7 +70,7 @@ let sha256 = "0md7rhw1ix4fp31bql9scvl4jpijixczm2ky7mxffwq3srvxvc5q"; }; - beamDeps = []; + beamDeps = [ ]; }; enough = buildRebar3 rec { @@ -83,7 +83,7 @@ let sha256 = "18gr9cvjar9rrmcj0crgwjb4np4adfbwcaxijajhwpjzvamwfq04"; }; - beamDeps = []; + beamDeps = [ ]; }; jason = buildMix rec { @@ -96,7 +96,7 @@ let sha256 = "0891p2yrg3ri04p302cxfww3fi16pvvw1kh4r91zg85jhl87k8vr"; }; - beamDeps = []; + beamDeps = [ ]; }; mime = buildMix rec { @@ -109,7 +109,7 @@ let sha256 = "0szzdfalafpawjrrwbrplhkgxjv8837mlxbkpbn5xlj4vgq0p8r7"; }; - beamDeps = []; + beamDeps = [ ]; }; plug = buildMix rec { @@ -148,7 +148,7 @@ let sha256 = "0hnqgzc3zas7j7wycgnkkdhaji5farkqccy2n4p1gqj5ccfrlm16"; }; - beamDeps = []; + beamDeps = [ ]; }; ranch = buildRebar3 rec { @@ -161,7 +161,7 @@ let sha256 = "1rfz5ld54pkd2w25jadyznia2vb7aw9bclck21fizargd39wzys9"; }; - beamDeps = []; + beamDeps = [ ]; }; systemd = buildRebar3 rec { @@ -187,8 +187,9 @@ let sha256 = "1mgyx9zw92g6w8fp9pblm3b0bghwxwwcbslrixq23ipzisfwxnfs"; }; - beamDeps = []; + beamDeps = [ ]; }; }; -in self +in +self diff --git a/priv/repo/migrations/20230813173233_create_posts.exs b/priv/repo/migrations/20230813173233_create_posts.exs new file mode 100644 index 0000000..c0e2f0f --- /dev/null +++ b/priv/repo/migrations/20230813173233_create_posts.exs @@ -0,0 +1,15 @@ +defmodule BdfrBrowser.Repo.Migrations.CreatePosts do + use Ecto.Migration + + def change do + create table(:posts) do + add :title, :string, size: 1024 + add :selftext, :text + add :url, :string, size: 2048 + add :permalink, :string, size: 2048 + add :author, :string + add :upvote_ratio, :float + add :posted_at, :utc_datetime + end + end +end diff --git a/priv/repo/migrations/20230813175025_create_subreddits.exs b/priv/repo/migrations/20230813175025_create_subreddits.exs new file mode 100644 index 0000000..8e2bd80 --- /dev/null +++ b/priv/repo/migrations/20230813175025_create_subreddits.exs @@ -0,0 +1,17 @@ +defmodule BdfrBrowser.Repo.Migrations.CreateSubreddits do + use Ecto.Migration + + def change do + create table(:subreddits, primary_key: false) do + add :id, :bigserial, primary_key: true + add :name, :string, size: 1024, unique: true + end + + alter table("posts") do + add :subreddit_id, references(:subreddits, type: :bigserial) + end + + create index("posts", :subreddit_id) + create index("subreddits", :name, unique: true) + end +end diff --git a/priv/repo/migrations/20230813194606_create_comments.exs b/priv/repo/migrations/20230813194606_create_comments.exs new file mode 100644 index 0000000..fdaee59 --- /dev/null +++ b/priv/repo/migrations/20230813194606_create_comments.exs @@ -0,0 +1,17 @@ +defmodule BdfrBrowser.Repo.Migrations.CreateComments do + use Ecto.Migration + + def change do + create table(:comments) do + add :author, :string + add :body, :text + add :score, :integer + add :posted_at, :utc_datetime + add :post_id, references(:posts) + add :parent_id, references(:comments) + end + + create index("comments", :post_id) + create index("comments", :parent_id) + end +end diff --git a/priv/repo/migrations/20230813215610_add_post_filename.exs b/priv/repo/migrations/20230813215610_add_post_filename.exs new file mode 100644 index 0000000..51a5c6e --- /dev/null +++ b/priv/repo/migrations/20230813215610_add_post_filename.exs @@ -0,0 +1,9 @@ +defmodule BdfrBrowser.Repo.Migrations.AddPostFilename do + use Ecto.Migration + + def change do + alter table("posts") do + add :filename, :string, size: 2048 + end + end +end diff --git a/priv/templates/http/_comment.eex b/priv/templates/http/_comment.eex index cdb57ae..bdecf30 100644 --- a/priv/templates/http/_comment.eex +++ b/priv/templates/http/_comment.eex @@ -1,4 +1,4 @@ -
+
@@ -10,7 +10,7 @@
<%= comment.author %>, - <%= trunc(comment.created_utc) |> DateTime.from_unix!() |> DateTime.to_iso8601() %> + <%= DateTime.to_iso8601(comment.posted_at) %>
@@ -18,6 +18,6 @@
-<%= for reply <- comment.replies do %> +<%= for reply <- BdfrBrowser.Repo.preload(comment, :children).children do %> <%= EEx.eval_file(comment_template, comment: reply, level: level + 1, comment_template: comment_template) %> <% end %> diff --git a/priv/templates/http/post.eex b/priv/templates/http/post.eex index f32f4db..3e0f769 100644 --- a/priv/templates/http/post.eex +++ b/priv/templates/http/post.eex @@ -17,7 +17,7 @@

<%= post.author %> - - <%= trunc(post.created_utc) |> DateTime.from_unix!() |> DateTime.to_iso8601() %> + <%= DateTime.to_iso8601(post.posted_at) %> - Open reddit

diff --git a/priv/templates/http/subreddit_posts.eex b/priv/templates/http/subreddit_posts.eex index 4c65c39..93c733c 100644 --- a/priv/templates/http/subreddit_posts.eex +++ b/priv/templates/http/subreddit_posts.eex @@ -19,9 +19,9 @@ <%= for post <- posts do %>
-
<%= post.title %>
+
<%= post.title %>
- <%= post.num_comments %> comment(s) - <%= trunc(post.created_utc) |> DateTime.from_unix!() |> DateTime.to_iso8601() %> + <%= post.num_comments %> comment(s) - <%= DateTime.to_iso8601(post.posted_at) %>