feat: Support importing and viewing chats

config/runtime.exs

@@ -2,6 +2,7 @@ import Config
 
 config :bdfr_browser,
   base_directory: System.get_env("BDFR_BROWSER_BASE_DIRECTORY", "/nonexistant"),
+  chat_directory: System.get_env("BDFR_BROWSER_CHAT_DIRECTORY", "/nonexistant"),
   http_ip: to_charlist(System.get_env("BDFR_BROWSER_HTTP_IP", "127.0.0.1")),
   http_port: String.to_integer(System.get_env("BDFR_BROWSER_HTTP_PORT", "4040"))
 

lib/bdfr_browser/chat.ex

@@ -0,0 +1,24 @@
+defmodule BdfrBrowser.Chat do
+  use Ecto.Schema
+
+  import Ecto.Query, only: [from: 2]
+
+  alias BdfrBrowser.Message
+
+  @primary_key {:id, :string, autogenerate: false}
+
+  schema "chats" do
+    field :accounts, {:array, :string}
+
+    has_many :messages, Message
+  end
+
+  def listing do
+    from(c in __MODULE__,
+      left_join: m in assoc(c, :messages),
+      select: %{id: c.id, accounts: c.accounts, num_messages: count(m.id), latest_message: max(m.posted_at)},
+      order_by: [desc: max(m.posted_at)],
+      group_by: c.id
+    )
+  end
+end

lib/bdfr_browser/http/plug.ex

@@ -1,7 +1,7 @@
 defmodule BdfrBrowser.HTTP.Plug do
   use Plug.Router
 
-  alias BdfrBrowser.{Repo, Post, Subreddit}
+  alias BdfrBrowser.{Chat, Message, Repo, Post, Subreddit}
 
   plug :match
   plug :dispatch
@@ -64,6 +64,30 @@ defmodule BdfrBrowser.HTTP.Plug do
     |> send_resp(200, content)
   end
 
+  get "/chats" do
+    tpl_args = [chats: Chat.listing() |> Repo.all()]
+    content = render_template("chats", tpl_args)
+
+    conn
+    |> put_resp_header("content-type", "text/html; charset=utf-8")
+    |> send_resp(200, content)
+  end
+
+  get "/chats/:id" do
+    chat_record = Repo.get(Chat, id)
+
+    tpl_args = [
+      chat: chat_record,
+      messages: chat_record |> Message.listing() |> Repo.all()
+    ]
+
+    content = render_template("chat", tpl_args)
+
+    conn
+    |> put_resp_header("content-type", "text/html; charset=utf-8")
+    |> send_resp(200, content)
+  end
+
   get "/static/*path" do
     file_path = Application.app_dir(:bdfr_browser, Path.join("priv/static", path))
 

lib/bdfr_browser/importer.ex

@@ -3,7 +3,7 @@ defmodule BdfrBrowser.Importer do
 
   use GenServer
 
-  alias BdfrBrowser.{Comment, Post, Repo, Subreddit}
+  alias BdfrBrowser.{Chat, Comment, Message, Post, Repo, Subreddit}
 
   def start_link([]) do
     GenServer.start_link(__MODULE__, [], name: __MODULE__)
@@ -49,6 +49,22 @@ defmodule BdfrBrowser.Importer do
     List.flatten(result)
   end
 
+  def chats do
+    _ = Logger.info("Importing chats ...")
+
+    result =
+      for chat <- read_chats(directory_key: :chat_directory) do
+        _ = Logger.info("Importing chat `#{chat["id"]}' ...")
+
+        {:ok, chat_record} = import_chat(chat)
+        message_records = for message <- chat["messages"], do: import_message(message, chat_record)
+
+        {chat_record, List.flatten(message_records)}
+      end
+
+    List.flatten(result)
+  end
+
   def background_import do
     GenServer.cast(__MODULE__, :background_import)
   end
@@ -64,6 +80,7 @@ defmodule BdfrBrowser.Importer do
   def handle_cast(:background_import, state) do
     _ = subreddits()
     _ = posts_and_comments()
+    _ = chats()
     {:noreply, state}
   end
 
@@ -73,7 +90,8 @@ defmodule BdfrBrowser.Importer do
     paths = Keyword.get(args, :paths, [])
     extname = Keyword.get(args, :ext, "")
     sort = Keyword.get(args, :sort, :desc)
-    base_directory = Application.fetch_env!(:bdfr_browser, :base_directory)
+    directory_key = Keyword.get(args, :directory_key, :base_directory)
+    base_directory = Application.fetch_env!(:bdfr_browser, directory_key)
 
     [base_directory | paths]
     |> Path.join()
@@ -99,6 +117,45 @@ defmodule BdfrBrowser.Importer do
     Enum.sort_by(parsed_posts, fn p -> p["created_utc"] end, sort)
   end
 
+  defp read_chats(args) do
+    directory_key = Keyword.get(args, :directory_key, :chat_directory)
+    base_directory = Application.fetch_env!(:bdfr_browser, directory_key)
+
+    new_chats =
+      for chat <- list_folders([{:ext, ".json"} | args]) do
+        file_path = Path.join([base_directory, chat])
+        parsed = file_path |> File.read!() |> Jason.decode!()
+        Map.put(parsed, "filename", chat)
+      end
+
+    old_chats =
+      for chat <- list_folders([{:ext, ".json_lines"} | args]) do
+        file_path = Path.join([base_directory, chat])
+
+        messages =
+          file_path
+          |> File.stream!()
+          |> Stream.map(&String.trim/1)
+          |> Stream.map(fn line ->
+            {:ok, [author, date, message]} = Jason.decode(line)
+            formatted_date = date |> String.replace(" UTC", "Z") |> String.replace(" ", "T")
+
+            %{
+              "author" => author,
+              "timestamp" => formatted_date,
+              "content" => %{
+                "Message" => message
+              }
+            }
+          end)
+          |> Enum.to_list()
+
+        %{"id" => Path.basename(chat, ".json_lines"), "messages" => messages, "filename" => chat}
+      end
+
+    old_chats ++ new_chats
+  end
+
   defp import_post(post, subreddit) do
     id = post["id"]
 
@@ -142,4 +199,38 @@ defmodule BdfrBrowser.Importer do
 
     [parent] ++ children
   end
+
+  defp import_chat(chat) do
+    id = chat["id"]
+    accounts = for message <- chat["messages"], uniq: true, do: message["author"]
+
+    %Chat{
+      id: id,
+      accounts: accounts
+    }
+    |> Repo.insert(
+      on_conflict: [set: [id: id]],
+      conflict_target: :id
+    )
+  end
+
+  defp import_message(message, chat) do
+    id = :sha3_256 |> :crypto.hash([chat.id, message["timestamp"]]) |> Base.encode16(case: :lower)
+    {:ok, posted_at, 0} = DateTime.from_iso8601(message["timestamp"])
+
+    {:ok, message} =
+      %Message{
+        id: id,
+        author: message["author"],
+        message: message["content"]["Message"],
+        posted_at: posted_at,
+        chat: chat
+      }
+      |> Repo.insert(
+        on_conflict: [set: [id: id]],
+        conflict_target: :id
+      )
+
+    message
+  end
 end

lib/bdfr_browser/message.ex

@@ -0,0 +1,25 @@
+defmodule BdfrBrowser.Message do
+  use Ecto.Schema
+
+  import Ecto.Query, only: [from: 2]
+
+  alias BdfrBrowser.Chat
+
+  @primary_key {:id, :string, autogenerate: false}
+  @foreign_key_type :string
+
+  schema "messages" do
+    field :author, :string
+    field :message, :string
+    field :posted_at, :utc_datetime
+
+    belongs_to :chat, Chat
+  end
+
+  def listing(chat) do
+    from(m in __MODULE__,
+      where: m.chat_id == ^chat.id,
+      order_by: [asc: m.posted_at]
+    )
+  end
+end

priv/repo/migrations/20230814110852_create_chats.exs

@@ -0,0 +1,21 @@
+defmodule BdfrBrowser.Repo.Migrations.CreateChats do
+  use Ecto.Migration
+
+  def change do
+    create table(:chats, primary_key: false) do
+      add :id, :string, primary_key: true, size: 1024
+      add :accounts, {:array, :string}
+    end
+
+    create table(:messages, primary_key: false) do
+      add :id, :string, primary_key: true, size: 256
+      add :author, :string
+      add :message, :text
+      add :posted_at, :utc_datetime
+
+      add :chat_id, references(:chats, type: :string)
+    end
+
+    create index("messages", :chat_id)
+  end
+end

priv/templates/http/chat.eex

@@ -0,0 +1,18 @@
+<h2>Chats</h2>
+
+<%= for message <- messages do %>
+  <div class="row" style="margin-bottom: 2px;">
+    <div class="card">
+      <div class="card-body" style="padding: 8px;">
+        <blockquote class="blockquote mb-0" style="font-size: 1rem;">
+          <%= Earmark.as_html!(message.message) %>
+
+          <footer class="blockquote-footer">
+            <%= message.author %>,
+            <small><%= DateTime.to_iso8601(message.posted_at) %></small>
+          </footer>
+        </blockquote>
+      </div>
+    </div>
+  </div>
+<% end %>

priv/templates/http/chats.eex

@@ -0,0 +1,16 @@
+<h2>Chats</h2>
+
+<div class="row text-center">
+  <div class="d-grid gap-2 col-12 mx-auto">
+    <%= for chat <- chats do %>
+      <div class="card">
+        <div class="card-body">
+          <h5 class="card-title"><a href="/chats/<%= URI.encode(chat.id, &URI.char_unreserved?/1) %>"><%= Enum.join(chat.accounts, ", ") %></a></h5>
+          <h6 class="card-subtitle mb-2 text-body-secondary">
+            <%= chat.num_messages %> message(s) - <%= DateTime.to_iso8601(chat.latest_message) %>
+          </h6>
+        </div>
+      </div>
+    <% end %>
+  </div>
+</div>

priv/templates/http/index.eex

@@ -2,6 +2,8 @@
 
 <div class="row text-center">
   <div class="d-grid gap-2 col-12 mx-auto">
+    <a class="btn btn-outline-primary btn-lg" href="/chats" role="button">Chats</a>
+
     <%= for subreddit <- subreddits do %>
       <a class="btn btn-outline-secondary btn-lg" href="/r/<%= subreddit %>" role="button"><%= subreddit %></a>
     <% end %>