feat: Postgres backend

.formatter.exs

@@ -4,5 +4,5 @@
     "{config,lib,test}/**/*.{ex,exs}"
   ],
   line_length: 120,
-  import_deps: [:plug]
+  import_deps: [:ecto, :ecto_sql, :plug]
 ]

.gitignore

@@ -19,6 +19,7 @@ erl_crash.dump
 # nix
 /.direnv
 /.elixir_ls
+.pre-commit-config.yaml
 
 # Exclude releases
 /rel/*

config/config.exs

@@ -1,7 +1,16 @@
 import Config
 
+config :bdfr_browser,
+  ecto_repos: [BdfrBrowser.Repo]
+
+config :bdfr_browser, BdfrBrowser.Repo,
+  migration_primary_key: [name: :id, type: :string],
+  migration_foreign_key: [column: :id, type: :string]
+
 config :logger,
   backends: [],
-  level: :warning,
+  level: :info,
   handle_otp_reports: false,
   handle_sasl_reports: false
+
+import_config "#{Mix.env()}.exs"

config/dev.exs

@@ -0,0 +1,5 @@
+import Config
+
+config :logger,
+  backends: [:console],
+  level: :debug

config/prod.exs

@@ -0,0 +1 @@
+import Config

config/runtime.exs

@@ -4,3 +4,9 @@ config :bdfr_browser,
   base_directory: System.get_env("BDFR_BROWSER_BASE_DIRECTORY", "/nonexistant"),
   http_ip: to_charlist(System.get_env("BDFR_BROWSER_HTTP_IP", "127.0.0.1")),
   http_port: String.to_integer(System.get_env("BDFR_BROWSER_HTTP_PORT", "4040"))
+
+config :bdfr_browser, BdfrBrowser.Repo,
+  database: System.get_env("BDFR_BROWSER_REPO_DATABASE", "postgres"),
+  username: System.get_env("BDFR_BROWSER_REPO_USER", "bdfr-browser"),
+  password: System.get_env("BDFR_BROWSER_REPO_PASSWORD", ""),
+  hostname: System.get_env("BDFR_BROWSER_REPO_HOST", "localhost")

config/test.exs

@@ -0,0 +1 @@
+import Config

flake.lock

@@ -1,15 +1,31 @@
 {
   "nodes": {
+    "flake-compat": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1673956053,
+        "narHash": "sha256-4gtG9iQuiKITOjNQQeQIpoIB6b16fm+504Ch3sNKLd8=",
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "rev": "35bb57c0c8d8b62bbfd284272c928ceb64ddbde9",
+        "type": "github"
+      },
+      "original": {
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "type": "github"
+      }
+    },
     "flake-parts": {
       "inputs": {
         "nixpkgs-lib": "nixpkgs-lib"
       },
       "locked": {
-        "lastModified": 1683560683,
+        "lastModified": 1690933134,
-        "narHash": "sha256-XAygPMN5Xnk/W2c1aW0jyEa6lfMDZWlQgiNtmHXytPc=",
+        "narHash": "sha256-ab989mN63fQZBFrkk4Q8bYxQCktuHmBIBqUG1jl6/FQ=",
         "owner": "hercules-ci",
         "repo": "flake-parts",
-        "rev": "006c75898cf814ef9497252b022e91c946ba8e17",
+        "rev": "59cf3f1447cfc75087e7273b04b31e689a8599fb",
         "type": "github"
       },
       "original": {
@@ -18,13 +34,66 @@
         "type": "github"
       }
     },
+    "flake-root": {
+      "locked": {
+        "lastModified": 1680964220,
+        "narHash": "sha256-dIdTYcf+KW9a4pKHsEbddvLVSfR1yiAJynzg2x0nfWg=",
+        "owner": "srid",
+        "repo": "flake-root",
+        "rev": "f1c0b93d05bdbea6c011136ba1a135c80c5b326c",
+        "type": "github"
+      },
+      "original": {
+        "owner": "srid",
+        "repo": "flake-root",
+        "type": "github"
+      }
+    },
+    "flake-utils": {
+      "inputs": {
+        "systems": "systems"
+      },
+      "locked": {
+        "lastModified": 1685518550,
+        "narHash": "sha256-o2d0KcvaXzTrPRIo0kOLV0/QXHhDQ5DTi+OxcjO8xqY=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "a1720a10a6cfe8234c0e93907ffe81be440f4cef",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
+    "gitignore": {
+      "inputs": {
+        "nixpkgs": [
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1660459072,
+        "narHash": "sha256-8DFJjXG8zqoONA1vXtgeKXy68KdJL5UaXR8NtVMUbx8=",
+        "owner": "hercules-ci",
+        "repo": "gitignore.nix",
+        "rev": "a20de23b925fd8264fd7fad6454652e142fd7f73",
+        "type": "github"
+      },
+      "original": {
+        "owner": "hercules-ci",
+        "repo": "gitignore.nix",
+        "type": "github"
+      }
+    },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1684242266,
+        "lastModified": 1691853136,
-        "narHash": "sha256-uaCQ2k1bmojHKjWQngvnnnxQJMY8zi1zq527HdWgQf8=",
+        "narHash": "sha256-wTzDsRV4HN8A2Sl0SVQY0q8ILs90CD43Ha//7gNZE+E=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "7e0743a5aea1dc755d4b761daf75b20aa486fdad",
+        "rev": "f0451844bbdf545f696f029d1448de4906c7f753",
         "type": "github"
       },
       "original": {
@@ -37,11 +106,11 @@
     "nixpkgs-lib": {
       "locked": {
         "dir": "lib",
-        "lastModified": 1682879489,
+        "lastModified": 1690881714,
-        "narHash": "sha256-sASwo8gBt7JDnOOstnps90K1wxmVfyhsTPPNTGBPjjg=",
+        "narHash": "sha256-h/nXluEqdiQHs1oSgkOOWF+j8gcJMWhwnZ9PFabN6q0=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "da45bf6ec7bbcc5d1e14d3795c025199f28e0de0",
+        "rev": "9e1960bc196baf6881340d53dccb203a951745a2",
         "type": "github"
       },
       "original": {
@@ -52,10 +121,107 @@
         "type": "github"
       }
     },
+    "nixpkgs-stable": {
+      "locked": {
+        "lastModified": 1685801374,
+        "narHash": "sha256-otaSUoFEMM+LjBI1XL/xGB5ao6IwnZOXc47qhIgJe8U=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "c37ca420157f4abc31e26f436c1145f8951ff373",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "ref": "nixos-23.05",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "pre-commit-hooks-nix": {
+      "inputs": {
+        "flake-compat": "flake-compat",
+        "flake-utils": "flake-utils",
+        "gitignore": [
+          "gitignore"
+        ],
+        "nixpkgs": [
+          "nixpkgs"
+        ],
+        "nixpkgs-stable": "nixpkgs-stable"
+      },
+      "locked": {
+        "lastModified": 1691747570,
+        "narHash": "sha256-J3fnIwJtHVQ0tK2JMBv4oAmII+1mCdXdpeCxtIsrL2A=",
+        "owner": "cachix",
+        "repo": "pre-commit-hooks.nix",
+        "rev": "c5ac3aa3324bd8aebe8622a3fc92eeb3975d317a",
+        "type": "github"
+      },
+      "original": {
+        "owner": "cachix",
+        "repo": "pre-commit-hooks.nix",
+        "type": "github"
+      }
+    },
+    "process-compose-flake": {
+      "locked": {
+        "lastModified": 1691182187,
+        "narHash": "sha256-dmHKgFXstdfX7nfnpbzR5H4DWGdWo610zsW9BCtI2WQ=",
+        "owner": "Platonic-Systems",
+        "repo": "process-compose-flake",
+        "rev": "bacdaf54ffe3a2c1734fd973a95e6b39b1560c2e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "Platonic-Systems",
+        "repo": "process-compose-flake",
+        "type": "github"
+      }
+    },
     "root": {
       "inputs": {
         "flake-parts": "flake-parts",
-        "nixpkgs": "nixpkgs"
+        "flake-root": "flake-root",
+        "gitignore": "gitignore",
+        "nixpkgs": "nixpkgs",
+        "pre-commit-hooks-nix": "pre-commit-hooks-nix",
+        "process-compose-flake": "process-compose-flake",
+        "treefmt-nix": "treefmt-nix"
+      }
+    },
+    "systems": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
+      }
+    },
+    "treefmt-nix": {
+      "inputs": {
+        "nixpkgs": [
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1691833704,
+        "narHash": "sha256-ASGhgGduEgcD3gQZhGr8xtmZ3PlVY+m2HuPnIZDbu78=",
+        "owner": "numtide",
+        "repo": "treefmt-nix",
+        "rev": "19dee4bf6001849006a63f3435247316b0488e99",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "treefmt-nix",
+        "type": "github"
       }
     }
   },

flake.nix

@@ -1,47 +1,193 @@
 {
   description = "bdfr-browser development environment";
+
   inputs = {
     nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable";
+
+    # Tools
+
     flake-parts.url = "github:hercules-ci/flake-parts";
+
+    flake-root.url = "github:srid/flake-root";
+
+    treefmt-nix = {
+      url = "github:numtide/treefmt-nix";
+      inputs.nixpkgs.follows = "nixpkgs";
+    };
+
+    gitignore = {
+      url = "github:hercules-ci/gitignore.nix";
+      inputs.nixpkgs.follows = "nixpkgs";
+    };
+
+    pre-commit-hooks-nix = {
+      url = "github:cachix/pre-commit-hooks.nix";
+      inputs.nixpkgs.follows = "nixpkgs";
+      inputs.gitignore.follows = "gitignore";
+    };
+
+    process-compose-flake.url = "github:Platonic-Systems/process-compose-flake";
   };
 
-  outputs = inputs@{ flake-parts, ... }:
+  outputs = inputs@{ flake-parts, gitignore, ... }:
     flake-parts.lib.mkFlake { inherit inputs; } {
       systems = [ "aarch64-darwin" "x86_64-linux" "aarch64-linux" ];
 
-      perSystem = { pkgs, lib, self', ... }:
+      imports = [
+        inputs.flake-root.flakeModule
+        inputs.treefmt-nix.flakeModule
+        inputs.pre-commit-hooks-nix.flakeModule
+        inputs.process-compose-flake.flakeModule
+      ];
+
+      perSystem = { pkgs, lib, config, self', ... }:
         let
           pname = "bdfr-browser";
           version = "0.0.1";
 
-          erlang = pkgs.beam.interpreters.erlangR25;
+          erlang = pkgs.beam.interpreters.erlangR26;
-          beamPackages = pkgs.beam.packagesWith erlang;
+          beamPackagesPrev = pkgs.beam.packagesWith erlang;
-          elixir = beamPackages.elixir_1_14;
+          elixir = beamPackagesPrev.elixir_1_15;
+
+          beamPackages = beamPackagesPrev // rec {
+            inherit erlang elixir;
+            hex = beamPackagesPrev.hex.override { inherit elixir; };
+            buildMix = beamPackagesPrev.buildMix.override { inherit elixir erlang hex; };
+            mixRelease = beamPackagesPrev.mixRelease.override { inherit erlang elixir; };
+          };
+
+          postgres = pkgs.postgresql_15;
 
           inherit (pkgs.stdenv) isDarwin;
+          inherit (gitignore.lib) gitignoreSource;
         in
         {
+          treefmt = {
+            inherit (config.flake-root) projectRootFile;
+            flakeCheck = false;
+
+            programs = {
+              mix-format = {
+                enable = true;
+                package = elixir;
+              };
+
+              nixpkgs-fmt.enable = true;
+            };
+          };
+
+          pre-commit = {
+            check.enable = false;
+
+            settings = {
+              excludes = [ "mix.nix" ];
+
+              hooks = {
+                deadnix.enable = true;
+                statix.enable = true;
+                treefmt.enable = true;
+              };
+            };
+          };
+
           devShells.default = pkgs.mkShell {
-            packages = (with pkgs; [
+            name = pname;
+
+            nativeBuildInputs = [
               erlang
               elixir
+              postgres
+            ] ++ lib.optionals isDarwin (with pkgs.darwin.apple_sdk.frameworks;
+              [
+                CoreFoundation
+                CoreServices
+              ]);
 
-              beamPackages.elixir-ls
+            packages = [
-              mix2nix
+              pkgs.mix2nix
-            ]) ++ lib.optionals isDarwin (with pkgs.darwin.apple_sdk.frameworks; [
+              self'.packages.bdfr-browser-dev
-              CoreFoundation
+            ];
-              CoreServices
+
-            ]);
+            inputsFrom = [
+              config.flake-root.devShell
+              config.treefmt.build.devShell
+              config.pre-commit.devShell
+            ];
 
             ERL_INCLUDE_PATH = "${erlang}/lib/erlang/usr/include";
+            TREEFMT_CONFIG_FILE = config.treefmt.build.configFile;
           };
 
           packages.default = beamPackages.mixRelease {
             inherit pname version;
 
-            src = ./.;
+            src = gitignoreSource ./.;
             mixNixDeps = import ./mix.nix { inherit lib beamPackages; };
           };
+
+          process-compose."${pname}-dev" =
+            let
+              db-host = "127.0.0.1";
+              db-user = "bdfr-browser";
+            in
+            {
+              port = 18808;
+
+              settings = {
+                environment = {
+                  BDFR_BROWSER_BASE_DIRECTORY = "/Volumes/MediaScraper/Reddit";
+                  BDFR_BROWSER_REPO_USER = db-user;
+                  BDFR_BROWSER_REPO_HOST = db-host;
+                };
+
+                processes = {
+                  db-init.command = ''
+                    if [ ! -d "$PWD/.direnv/postgres/data" ]; then
+                      echo "Initializing database ..."
+                      mkdir -p "$PWD/.direnv/postgres"
+                      ${postgres}/bin/initdb --username ${db-user} --pgdata "$PWD/.direnv/postgres/data" --auth trust
+                    else
+                      echo "Database already initialized"
+                    fi
+                  '';
+
+                  db = {
+                    command = "${postgres}/bin/postgres -D $PWD/.direnv/postgres/data";
+
+                    depends_on.db-init.condition = "process_completed_successfully";
+
+                    readiness_probe.exec.command = "PGCONNECT_TIMEOUT=1 ${postgres}/bin/psql -h ${db-host} -U ${db-user} -l";
+                  };
+
+                  app-setup.command = ''
+                    mix local.hex --if-missing --force
+                    mix local.rebar --force
+                    mix deps.get
+                  '';
+
+                  app-compile = {
+                    command = "mix release --overwrite";
+
+                    depends_on.app-setup.condition = "process_completed_successfully";
+                  };
+
+                  app = {
+                    command = "$PWD/_build/dev/rel/bdfr_browser/bin/bdfr_browser start";
+
+                    depends_on = {
+                      db.condition = "process_healthy";
+                      app-compile.condition = "process_completed_successfully";
+                    };
+
+                    readiness_probe.http_get = {
+                      host = "127.0.0.1";
+                      port = 4040;
+                      path = "/_ping";
+                    };
+                  };
+                };
+              };
+            };
         };
     };
 }

lib/bdfr_browser/application.ex

@@ -7,10 +7,14 @@ defmodule BdfrBrowser.Application do
 
   @impl true
   def start(_type, _args) do
+    repos = Application.fetch_env!(:bdfr_browser, :ecto_repos)
     {:ok, http_ip} = :inet.parse_address(Application.fetch_env!(:bdfr_browser, :http_ip))
     http_port = Application.fetch_env!(:bdfr_browser, :http_port)
 
     children = [
+      {Ecto.Migrator, repos: repos, skip: System.get_env("SKIP_MIGRATIONS") == "true"},
+      BdfrBrowser.Repo,
+      BdfrBrowser.Importer,
       {Plug.Cowboy, scheme: :http, plug: BdfrBrowser.HTTP.Plug, options: [ip: http_ip, port: http_port]},
       :systemd.ready()
     ]

lib/bdfr_browser/comment.ex

@@ -0,0 +1,19 @@
+defmodule BdfrBrowser.Comment do
+  use Ecto.Schema
+
+  alias BdfrBrowser.Post
+
+  @primary_key {:id, :string, autogenerate: false}
+  @foreign_key_type :string
+
+  schema "comments" do
+    field :author, :string
+    field :body, :string
+    field :score, :integer
+    field :posted_at, :utc_datetime
+
+    belongs_to :post, Post
+    belongs_to :parent, __MODULE__
+    has_many :children, __MODULE__, foreign_key: :parent_id
+  end
+end

lib/bdfr_browser/http/plug.ex

@@ -1,12 +1,14 @@
 defmodule BdfrBrowser.HTTP.Plug do
   use Plug.Router
 
+  alias BdfrBrowser.{Repo, Post, Subreddit}
+
   plug :match
   plug :dispatch
 
   get "/" do
     tpl_params = [
-      subreddits: list_folders(sort: :asc)
+      subreddits: Subreddit.names() |> Repo.all()
     ]
 
     tpl_file = Application.app_dir(:bdfr_browser, "priv/templates/http/index.eex")
@@ -18,9 +20,11 @@ defmodule BdfrBrowser.HTTP.Plug do
   end
 
   get "/r/:subreddit" do
+    subreddit_record = Repo.get_by(Subreddit, name: subreddit)
+
     tpl_params = [
       subreddit: subreddit,
-      dates: list_folders(paths: [subreddit])
+      dates: subreddit_record |> Post.date_listing() |> Repo.all()
     ]
 
     tpl_file = Application.app_dir(:bdfr_browser, "priv/templates/http/subreddit.eex")
@@ -32,10 +36,12 @@ defmodule BdfrBrowser.HTTP.Plug do
   end
 
   get "/r/:subreddit/:date" do
+    subreddit_record = Repo.get_by(Subreddit, name: subreddit)
+
     tpl_params = [
       subreddit: subreddit,
       date: date,
-      posts: read_posts(paths: [subreddit, date], ext: ".json")
+      posts: subreddit_record |> Post.during_month(date) |> Repo.all()
     ]
 
     tpl_file = Application.app_dir(:bdfr_browser, "priv/templates/http/subreddit_posts.eex")
@@ -46,12 +52,14 @@ defmodule BdfrBrowser.HTTP.Plug do
     |> send_resp(200, content)
   end
 
-  get "/r/:subreddit/:date/:post" do
+  get "/r/:subreddit/:date/:id" do
+    post_record = Post |> Repo.get(id) |> Repo.preload(comments: :children)
+
     tpl_params = [
       subreddit: subreddit,
       date: date,
-      post: read_post(post, paths: [subreddit, date]),
+      post: post_record,
-      media: post_media(post, paths: [subreddit, date]),
+      media: post_media(post_record.filename, paths: [subreddit, date]),
       comment_template: Application.app_dir(:bdfr_browser, "priv/templates/http/_comment.eex")
     ]
 
@@ -73,66 +81,21 @@ defmodule BdfrBrowser.HTTP.Plug do
     |> send_resp(200, media)
   end
 
+  get "/_import" do
+    :ok = BdfrBrowser.Importer.background_import()
+    send_resp(conn, 200, "IMPORTING")
+  end
+
+  get "/_ping" do
+    send_resp(conn, 200, "PONG")
+  end
+
   match _ do
     send_resp(conn, 404, "Not Found")
   end
 
   # Helper
 
-  defp list_folders(args) do
-    paths = Keyword.get(args, :paths, [])
-    extname = Keyword.get(args, :ext, "")
-    sort = Keyword.get(args, :sort, :desc)
-    base_directory = Application.fetch_env!(:bdfr_browser, :base_directory)
-
-    [base_directory | paths]
-    |> Path.join()
-    |> File.ls!()
-    |> Enum.filter(fn s -> not String.starts_with?(s, ".") and Path.extname(s) == extname end)
-    |> Enum.sort_by(&String.downcase/1, sort)
-  end
-
-  defp read_posts(args) do
-    posts = list_folders(args)
-    sort = Keyword.get(args, :sort, :desc)
-
-    base_directory = Application.fetch_env!(:bdfr_browser, :base_directory)
-    post_dir = Path.join([base_directory | Keyword.fetch!(args, :paths)])
-
-    compact_posts =
-      for post <- posts do
-        {:ok, content} = [post_dir, post] |> Path.join() |> File.read!() |> Jason.decode()
-
-        %{
-          title: extract_title(content["title"]),
-          author: content["author"],
-          num_comments: content["num_comments"],
-          created_utc: content["created_utc"],
-          filename: Path.basename(post, ".json")
-        }
-      end
-
-    Enum.sort_by(compact_posts, fn p -> p.created_utc end, sort)
-  end
-
-  defp extract_title(title) do
-    if String.length(title) == 0 do
-      "Empty Title"
-    else
-      title
-    end
-  end
-
-  defp read_post(post, args) do
-    base_directory = Application.fetch_env!(:bdfr_browser, :base_directory)
-    post_dir = Path.join([base_directory | Keyword.fetch!(args, :paths)])
-    post_file = "#{post}.json"
-
-    {:ok, content} = [post_dir, post_file] |> Path.join() |> File.read!() |> Jason.decode(keys: :atoms)
-
-    content
-  end
-
   defp post_media(post, args) do
     base_directory = Application.fetch_env!(:bdfr_browser, :base_directory)
     post_dir = Path.join([base_directory | Keyword.fetch!(args, :paths)])

lib/bdfr_browser/importer.ex

@@ -0,0 +1,145 @@
+defmodule BdfrBrowser.Importer do
+  require Logger
+
+  use GenServer
+
+  alias BdfrBrowser.{Comment, Post, Repo, Subreddit}
+
+  def start_link([]) do
+    GenServer.start_link(__MODULE__, [], name: __MODULE__)
+  end
+
+  def subreddits do
+    _ = Logger.info("Importing subreddits ...")
+
+    folders = list_folders(sort: :asc)
+
+    for folder <- folders do
+      %Subreddit{name: folder}
+      |> Repo.insert(
+        on_conflict: :nothing,
+        conflict_target: :name
+      )
+    end
+  end
+
+  def posts_and_comments do
+    _ = Logger.info("Importing posts and comments ...")
+
+    result =
+      for subreddit <- list_folders(sort: :asc) do
+        _ = Logger.info("Importing entries from `#{subreddit}' ...")
+
+        subreddit_record = Repo.get_by(Subreddit, name: subreddit)
+
+        for date <- list_folders(paths: [subreddit]) do
+          _ = Logger.debug("Importing entries from `#{subreddit}' on `#{date}' ...")
+
+          for post <- read_posts(paths: [subreddit, date], ext: ".json") do
+            _ = Logger.debug("Importing `#{post["id"]}' from `#{subreddit}' ...")
+
+            {:ok, post_record} = import_post(post, subreddit_record)
+            comment_records = for comment <- post["comments"], do: import_comment(comment, post_record, nil)
+
+            {post_record, List.flatten(comment_records)}
+          end
+        end
+      end
+
+    List.flatten(result)
+  end
+
+  def background_import do
+    GenServer.cast(__MODULE__, :background_import)
+  end
+
+  # Callbacks
+
+  @impl true
+  def init([]) do
+    {:ok, nil}
+  end
+
+  @impl true
+  def handle_cast(:background_import, state) do
+    _ = subreddits()
+    _ = posts_and_comments()
+    {:noreply, state}
+  end
+
+  # Helper
+
+  defp list_folders(args) do
+    paths = Keyword.get(args, :paths, [])
+    extname = Keyword.get(args, :ext, "")
+    sort = Keyword.get(args, :sort, :desc)
+    base_directory = Application.fetch_env!(:bdfr_browser, :base_directory)
+
+    [base_directory | paths]
+    |> Path.join()
+    |> File.ls!()
+    |> Enum.filter(fn s -> not String.starts_with?(s, ".") and Path.extname(s) == extname end)
+    |> Enum.sort_by(&String.downcase/1, sort)
+  end
+
+  defp read_posts(args) do
+    posts = list_folders(args)
+    sort = Keyword.get(args, :sort, :desc)
+
+    base_directory = Application.fetch_env!(:bdfr_browser, :base_directory)
+    post_dir = Path.join([base_directory | Keyword.fetch!(args, :paths)])
+
+    parsed_posts =
+      for post <- posts do
+        file_path = Path.join([post_dir, post])
+        parsed = file_path |> File.read!() |> Jason.decode!()
+        Map.put(parsed, "filename", post)
+      end
+
+    Enum.sort_by(parsed_posts, fn p -> p["created_utc"] end, sort)
+  end
+
+  defp import_post(post, subreddit) do
+    id = post["id"]
+
+    %Post{
+      id: id,
+      title: post["title"],
+      selftext: post["selftext"],
+      url: post["url"],
+      permalink: post["permalink"],
+      author: post["author"],
+      upvote_ratio: post["upvote_ratio"],
+      posted_at: DateTime.from_unix!(trunc(post["created_utc"])),
+      filename: Path.basename(post["filename"], ".json"),
+      subreddit: subreddit
+    }
+    |> Repo.insert(
+      on_conflict: [set: [id: id]],
+      conflict_target: :id
+    )
+  end
+
+  defp import_comment(comment, post, parent) do
+    id = comment["id"]
+
+    {:ok, parent} =
+      %Comment{
+        id: id,
+        author: comment["author"],
+        body: comment["body"],
+        score: comment["score"],
+        posted_at: DateTime.from_unix!(trunc(comment["created_utc"])),
+        post: post,
+        parent: parent
+      }
+      |> Repo.insert(
+        on_conflict: [set: [id: id]],
+        conflict_target: :id
+      )
+
+    children = for child <- comment["replies"], do: import_comment(child, post, parent)
+
+    [parent] ++ children
+  end
+end

lib/bdfr_browser/post.ex

@@ -0,0 +1,49 @@
+defmodule BdfrBrowser.Post do
+  use Ecto.Schema
+
+  import Ecto.Query, only: [from: 2]
+
+  alias BdfrBrowser.{Comment, Subreddit}
+
+  @primary_key {:id, :string, autogenerate: false}
+
+  schema "posts" do
+    field :title, :string
+    field :selftext, :string
+    field :url, :string
+    field :permalink, :string
+    field :author, :string
+    field :upvote_ratio, :float
+    field :posted_at, :utc_datetime
+    field :filename, :string
+
+    belongs_to :subreddit, Subreddit
+    has_many :comments, Comment
+  end
+
+  def date_listing(subreddit) do
+    from(p in __MODULE__,
+      select: fragment("to_char(?, 'YYYY-MM')", p.posted_at),
+      where: p.subreddit_id == ^subreddit.id,
+      distinct: true,
+      order_by: [desc: fragment("to_char(?, 'YYYY-MM')", p.posted_at)]
+    )
+  end
+
+  def during_month(subreddit, month_str) do
+    {:ok, d} = Date.from_iso8601("#{month_str}-01")
+    during_range(subreddit, Date.beginning_of_month(d), Date.end_of_month(d))
+  end
+
+  def during_range(subreddit, start_date, end_date) do
+    from(p in __MODULE__,
+      left_join: c in assoc(p, :comments),
+      select: %{id: p.id, title: p.title, author: p.author, posted_at: p.posted_at, num_comments: count(c.id)},
+      where:
+        p.subreddit_id == ^subreddit.id and type(p.posted_at, :date) >= ^start_date and
+          type(p.posted_at, :date) <= ^end_date,
+      order_by: [desc: p.posted_at],
+      group_by: p.id
+    )
+  end
+end

lib/bdfr_browser/repo.ex

@@ -0,0 +1,5 @@
+defmodule BdfrBrowser.Repo do
+  use Ecto.Repo,
+    otp_app: :bdfr_browser,
+    adapter: Ecto.Adapters.Postgres
+end

lib/bdfr_browser/subreddit.ex

@@ -0,0 +1,17 @@
+defmodule BdfrBrowser.Subreddit do
+  use Ecto.Schema
+
+  import Ecto.Query, only: [from: 2]
+
+  alias BdfrBrowser.Post
+
+  schema "subreddits" do
+    field(:name, :string)
+
+    has_many(:posts, Post)
+  end
+
+  def names do
+    from(s in __MODULE__, select: s.name, order_by: [asc: s.name])
+  end
+end

mix.exs

@@ -21,6 +21,8 @@ defmodule BdfrBrowser.MixProject do
   defp deps do
     [
       {:plug_cowboy, "~> 2.6"},
+      {:ecto_sql, "~> 3.10"},
+      {:postgrex, "~> 0.17"},
       {:jason, "~> 1.4"},
       {:earmark, "~> 1.4"},
       {:systemd, "~> 0.6"}

mix.lock

@@ -2,14 +2,19 @@
   "cowboy": {:hex, :cowboy, "2.10.0", "ff9ffeff91dae4ae270dd975642997afe2a1179d94b1887863e43f681a203e26", [:make, :rebar3], [{:cowlib, "2.12.1", [hex: :cowlib, repo: "hexpm", optional: false]}, {:ranch, "1.8.0", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "3afdccb7183cc6f143cb14d3cf51fa00e53db9ec80cdcd525482f5e99bc41d6b"},
   "cowboy_telemetry": {:hex, :cowboy_telemetry, "0.4.0", "f239f68b588efa7707abce16a84d0d2acf3a0f50571f8bb7f56a15865aae820c", [:rebar3], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "7d98bac1ee4565d31b62d59f8823dfd8356a169e7fcbb83831b8a5397404c9de"},
   "cowlib": {:hex, :cowlib, "2.12.1", "a9fa9a625f1d2025fe6b462cb865881329b5caff8f1854d1cbc9f9533f00e1e1", [:make, :rebar3], [], "hexpm", "163b73f6367a7341b33c794c4e88e7dbfe6498ac42dcd69ef44c5bc5507c8db0"},
-  "earmark": {:hex, :earmark, "1.4.38", "ba8fda946c259c6e8f6759d3647d448e9216e2c0afed8c6ae7f8ce1f7072a497", [:mix], [{:earmark_parser, "~> 1.4.32", [hex: :earmark_parser, repo: "hexpm", optional: false]}], "hexpm", "f938e30de4167e7d8f3bf588b01dc041138278dda1e5a13fb9ec89b43dd5ec7f"},
+  "db_connection": {:hex, :db_connection, "2.5.0", "bb6d4f30d35ded97b29fe80d8bd6f928a1912ca1ff110831edcd238a1973652c", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "c92d5ba26cd69ead1ff7582dbb860adeedfff39774105a4f1c92cbb654b55aa2"},
-  "earmark_parser": {:hex, :earmark_parser, "1.4.32", "fa739a0ecfa34493de19426681b23f6814573faee95dfd4b4aafe15a7b5b32c6", [:mix], [], "hexpm", "b8b0dd77d60373e77a3d7e8afa598f325e49e8663a51bcc2b88ef41838cca755"},
+  "decimal": {:hex, :decimal, "2.1.1", "5611dca5d4b2c3dd497dec8f68751f1f1a54755e8ed2a966c2633cf885973ad6", [:mix], [], "hexpm", "53cfe5f497ed0e7771ae1a475575603d77425099ba5faef9394932b35020ffcc"},
+  "earmark": {:hex, :earmark, "1.4.39", "acdb2f02c536471029dbcc509fbd6b94b89f40ad7729fb3f68f4b6944843f01d", [:mix], [{:earmark_parser, "~> 1.4.33", [hex: :earmark_parser, repo: "hexpm", optional: false]}], "hexpm", "156c9d8ec3cbeccdbf26216d8247bdeeacc8c76b4d9eee7554be2f1b623ea440"},
+  "earmark_parser": {:hex, :earmark_parser, "1.4.33", "3c3fd9673bb5dcc9edc28dd90f50c87ce506d1f71b70e3de69aa8154bc695d44", [:mix], [], "hexpm", "2d526833729b59b9fdb85785078697c72ac5e5066350663e5be6a1182da61b8f"},
+  "ecto": {:hex, :ecto, "3.10.3", "eb2ae2eecd210b4eb8bece1217b297ad4ff824b4384c0e3fdd28aaf96edd6135", [:mix], [{:decimal, "~> 1.6 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "44bec74e2364d491d70f7e42cd0d690922659d329f6465e89feb8a34e8cd3433"},
+  "ecto_sql": {:hex, :ecto_sql, "3.10.1", "6ea6b3036a0b0ca94c2a02613fd9f742614b5cfe494c41af2e6571bb034dd94c", [:mix], [{:db_connection, "~> 2.4.1 or ~> 2.5", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.10.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.6.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.16.0 or ~> 0.17.0 or ~> 1.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:tds, "~> 2.1.1 or ~> 2.2", [hex: :tds, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "f6a25bdbbd695f12c8171eaff0851fa4c8e72eec1e98c7364402dda9ce11c56b"},
   "enough": {:hex, :enough, "0.1.0", "0254710c52d324e2dadde54cb56fbb80a792c2eb285669b8379efd0752bf89f0", [:rebar3], [], "hexpm", "0460c7abda5f5e0ea592b12bc6976b8a5c4b96e42f332059cd396525374bf9a1"},
-  "jason": {:hex, :jason, "1.4.0", "e855647bc964a44e2f67df589ccf49105ae039d4179db7f6271dfd3843dc27e6", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "79a3791085b2a0f743ca04cec0f7be26443738779d09302e01318f97bdb82121"},
+  "jason": {:hex, :jason, "1.4.1", "af1504e35f629ddcdd6addb3513c3853991f694921b1b9368b0bd32beb9f1b63", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "fbb01ecdfd565b56261302f7e1fcc27c4fb8f32d56eab74db621fc154604a7a1"},
-  "mime": {:hex, :mime, "2.0.3", "3676436d3d1f7b81b5a2d2bd8405f412c677558c81b1c92be58c00562bb59095", [:mix], [], "hexpm", "27a30bf0db44d25eecba73755acf4068cbfe26a4372f9eb3e4ea3a45956bff6b"},
+  "mime": {:hex, :mime, "2.0.5", "dc34c8efd439abe6ae0343edbb8556f4d63f178594894720607772a041b04b02", [:mix], [], "hexpm", "da0d64a365c45bc9935cc5c8a7fc5e49a0e0f9932a761c55d6c52b142780a05c"},
   "plug": {:hex, :plug, "1.14.2", "cff7d4ec45b4ae176a227acd94a7ab536d9b37b942c8e8fa6dfc0fff98ff4d80", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "842fc50187e13cf4ac3b253d47d9474ed6c296a8732752835ce4a86acdf68d13"},
   "plug_cowboy": {:hex, :plug_cowboy, "2.6.1", "9a3bbfceeb65eff5f39dab529e5cd79137ac36e913c02067dba3963a26efe9b2", [:mix], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:cowboy_telemetry, "~> 0.3", [hex: :cowboy_telemetry, repo: "hexpm", optional: false]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "de36e1a21f451a18b790f37765db198075c25875c64834bcc82d90b309eb6613"},
   "plug_crypto": {:hex, :plug_crypto, "1.2.5", "918772575e48e81e455818229bf719d4ab4181fcbf7f85b68a35620f78d89ced", [:mix], [], "hexpm", "26549a1d6345e2172eb1c233866756ae44a9609bd33ee6f99147ab3fd87fd842"},
+  "postgrex": {:hex, :postgrex, "0.17.2", "a3ec9e3239d9b33f1e5841565c4eb200055c52cc0757a22b63ca2d529bbe764c", [:mix], [{:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "80a918a9e9531d39f7bd70621422f3ebc93c01618c645f2d91306f50041ed90c"},
   "ranch": {:hex, :ranch, "1.8.0", "8c7a100a139fd57f17327b6413e4167ac559fbc04ca7448e9be9057311597a1d", [:make, :rebar3], [], "hexpm", "49fbcfd3682fab1f5d109351b61257676da1a2fdbe295904176d5e521a2ddfe5"},
   "systemd": {:hex, :systemd, "0.6.2", "aaa24f1e3e6cb978c45369768b1abd766a0dbff637ed61254ca64797bcec9963", [:rebar3], [{:enough, "~> 0.1.0", [hex: :enough, repo: "hexpm", optional: false]}], "hexpm", "5062b911800c1ab05157c7bf9a9fbe23dd24c58891c87fd12d2e3ed8fc1708b8"},
   "telemetry": {:hex, :telemetry, "1.2.1", "68fdfe8d8f05a8428483a97d7aab2f268aaff24b49e0f599faa091f1d4e7f61c", [:rebar3], [], "hexpm", "dad9ce9d8effc621708f99eac538ef1cbe05d6a874dd741de2e689c47feafed5"},

mix.nix

@@ -1,4 +1,4 @@
-{ lib, beamPackages, overrides ? (x: y: {}) }:
+{ lib, beamPackages, overrides ? (x: y: { }) }:
 
 let
   buildRebar3 = lib.makeOverridable beamPackages.buildRebar3;
@@ -44,7 +44,7 @@ let
         sha256 = "1c4dgi8canscyjgddp22mjc69znvwy44wk3r7jrl2wvs6vv76fqn";
       };
 
-      beamDeps = [];
+      beamDeps = [ ];
     };
 
     earmark = buildMix rec {
@@ -70,7 +70,7 @@ let
         sha256 = "0md7rhw1ix4fp31bql9scvl4jpijixczm2ky7mxffwq3srvxvc5q";
       };
 
-      beamDeps = [];
+      beamDeps = [ ];
     };
 
     enough = buildRebar3 rec {
@@ -83,7 +83,7 @@ let
         sha256 = "18gr9cvjar9rrmcj0crgwjb4np4adfbwcaxijajhwpjzvamwfq04";
       };
 
-      beamDeps = [];
+      beamDeps = [ ];
     };
 
     jason = buildMix rec {
@@ -96,7 +96,7 @@ let
         sha256 = "0891p2yrg3ri04p302cxfww3fi16pvvw1kh4r91zg85jhl87k8vr";
       };
 
-      beamDeps = [];
+      beamDeps = [ ];
     };
 
     mime = buildMix rec {
@@ -109,7 +109,7 @@ let
         sha256 = "0szzdfalafpawjrrwbrplhkgxjv8837mlxbkpbn5xlj4vgq0p8r7";
       };
 
-      beamDeps = [];
+      beamDeps = [ ];
     };
 
     plug = buildMix rec {
@@ -148,7 +148,7 @@ let
         sha256 = "0hnqgzc3zas7j7wycgnkkdhaji5farkqccy2n4p1gqj5ccfrlm16";
       };
 
-      beamDeps = [];
+      beamDeps = [ ];
     };
 
     ranch = buildRebar3 rec {
@@ -161,7 +161,7 @@ let
         sha256 = "1rfz5ld54pkd2w25jadyznia2vb7aw9bclck21fizargd39wzys9";
       };
 
-      beamDeps = [];
+      beamDeps = [ ];
     };
 
     systemd = buildRebar3 rec {
@@ -187,8 +187,9 @@ let
         sha256 = "1mgyx9zw92g6w8fp9pblm3b0bghwxwwcbslrixq23ipzisfwxnfs";
       };
 
-      beamDeps = [];
+      beamDeps = [ ];
     };
   };
-in self
+in
+self
 

priv/repo/migrations/20230813173233_create_posts.exs

@@ -0,0 +1,15 @@
+defmodule BdfrBrowser.Repo.Migrations.CreatePosts do
+  use Ecto.Migration
+
+  def change do
+    create table(:posts) do
+      add :title, :string, size: 1024
+      add :selftext, :text
+      add :url, :string, size: 2048
+      add :permalink, :string, size: 2048
+      add :author, :string
+      add :upvote_ratio, :float
+      add :posted_at, :utc_datetime
+    end
+  end
+end

priv/repo/migrations/20230813175025_create_subreddits.exs

@@ -0,0 +1,17 @@
+defmodule BdfrBrowser.Repo.Migrations.CreateSubreddits do
+  use Ecto.Migration
+
+  def change do
+    create table(:subreddits, primary_key: false) do
+      add :id, :bigserial, primary_key: true
+      add :name, :string, size: 1024, unique: true
+    end
+
+    alter table("posts") do
+      add :subreddit_id, references(:subreddits, type: :bigserial)
+    end
+
+    create index("posts", :subreddit_id)
+    create index("subreddits", :name, unique: true)
+  end
+end

priv/repo/migrations/20230813194606_create_comments.exs

@@ -0,0 +1,17 @@
+defmodule BdfrBrowser.Repo.Migrations.CreateComments do
+  use Ecto.Migration
+
+  def change do
+    create table(:comments) do
+      add :author, :string
+      add :body, :text
+      add :score, :integer
+      add :posted_at, :utc_datetime
+      add :post_id, references(:posts)
+      add :parent_id, references(:comments)
+    end
+
+    create index("comments", :post_id)
+    create index("comments", :parent_id)
+  end
+end

priv/repo/migrations/20230813215610_add_post_filename.exs

@@ -0,0 +1,9 @@
+defmodule BdfrBrowser.Repo.Migrations.AddPostFilename do
+  use Ecto.Migration
+
+  def change do
+    alter table("posts") do
+      add :filename, :string, size: 2048
+    end
+  end
+end

priv/templates/http/_comment.eex

@@ -1,4 +1,4 @@
-<div style="padding-left: <%= level * 5%>px;">
+<div style="padding-left: <%= level * 5 %>px;">
   <div class="card">
     <div class="card-body">
       <blockquote class="blockquote mb-0">
@@ -10,7 +10,7 @@
         
         <footer class="blockquote-footer">
           <%= comment.author %>,
-          <small><%= trunc(comment.created_utc) |> DateTime.from_unix!() |> DateTime.to_iso8601() %></small>
+          <small><%= DateTime.to_iso8601(comment.posted_at) %></small>
         </footer>
       </blockquote>
     </div>
@@ -18,6 +18,6 @@
   <br>
 </div>
 
-<%= for reply <- comment.replies do %>
+<%= for reply <- BdfrBrowser.Repo.preload(comment, :children).children do %>
   <%= EEx.eval_file(comment_template, comment: reply, level: level + 1, comment_template: comment_template) %>
 <% end %>

priv/templates/http/post.eex

@@ -17,7 +17,7 @@
       <p>
         <small><a href="https://reddit.com/user/<%= post.author %>"><%= post.author %></a></small>
         -
-        <small><%= trunc(post.created_utc) |> DateTime.from_unix!() |> DateTime.to_iso8601() %></small>
+        <small><%= DateTime.to_iso8601(post.posted_at) %></small>
         -
         <a href="https://reddit.com<%= post.permalink %>">Open reddit</a>
       </p>

priv/templates/http/subreddit_posts.eex

@@ -19,9 +19,9 @@
           <%= for post <- posts do %>
             <div class="card">
               <div class="card-body">
-                <h5 class="card-title"><a href="/r/<%= subreddit %>/<%= date %>/<%= URI.encode(post.filename, &URI.char_unreserved?/1) %>"><%= post.title %></a></h5>
+                <h5 class="card-title"><a href="/r/<%= subreddit %>/<%= date %>/<%= post.id %>"><%= post.title %></a></h5>
                 <h6 class="card-subtitle mb-2 text-body-secondary">
-                  <%= post.num_comments %> comment(s) - <%= trunc(post.created_utc) |> DateTime.from_unix!() |> DateTime.to_iso8601() %>
+                  <%= post.num_comments %> comment(s) - <%= DateTime.to_iso8601(post.posted_at) %>
                 </h6>
               </div>
             </div>