feat: Support importing and viewing chats

This commit is contained in:
Daniel Kempkens 2023-08-14 15:05:27 +02:00
parent 82e435ddd8
commit 23a1d6a9d6
Signed by: daniel
SSH key fingerprint: SHA256:Ks/MyhQYcPRQiwMKLAKquWCdCPe3JXlb1WttgnAoSeM
9 changed files with 225 additions and 3 deletions

View file

@ -2,6 +2,7 @@ import Config
config :bdfr_browser, config :bdfr_browser,
base_directory: System.get_env("BDFR_BROWSER_BASE_DIRECTORY", "/nonexistant"), base_directory: System.get_env("BDFR_BROWSER_BASE_DIRECTORY", "/nonexistant"),
chat_directory: System.get_env("BDFR_BROWSER_CHAT_DIRECTORY", "/nonexistant"),
http_ip: to_charlist(System.get_env("BDFR_BROWSER_HTTP_IP", "127.0.0.1")), http_ip: to_charlist(System.get_env("BDFR_BROWSER_HTTP_IP", "127.0.0.1")),
http_port: String.to_integer(System.get_env("BDFR_BROWSER_HTTP_PORT", "4040")) http_port: String.to_integer(System.get_env("BDFR_BROWSER_HTTP_PORT", "4040"))

24
lib/bdfr_browser/chat.ex Normal file
View file

@ -0,0 +1,24 @@
defmodule BdfrBrowser.Chat do
use Ecto.Schema
import Ecto.Query, only: [from: 2]
alias BdfrBrowser.Message
@primary_key {:id, :string, autogenerate: false}
schema "chats" do
field :accounts, {:array, :string}
has_many :messages, Message
end
def listing do
from(c in __MODULE__,
left_join: m in assoc(c, :messages),
select: %{id: c.id, accounts: c.accounts, num_messages: count(m.id), latest_message: max(m.posted_at)},
order_by: [desc: max(m.posted_at)],
group_by: c.id
)
end
end

View file

@ -1,7 +1,7 @@
defmodule BdfrBrowser.HTTP.Plug do defmodule BdfrBrowser.HTTP.Plug do
use Plug.Router use Plug.Router
alias BdfrBrowser.{Repo, Post, Subreddit} alias BdfrBrowser.{Chat, Message, Repo, Post, Subreddit}
plug :match plug :match
plug :dispatch plug :dispatch
@ -64,6 +64,30 @@ defmodule BdfrBrowser.HTTP.Plug do
|> send_resp(200, content) |> send_resp(200, content)
end end
get "/chats" do
tpl_args = [chats: Chat.listing() |> Repo.all()]
content = render_template("chats", tpl_args)
conn
|> put_resp_header("content-type", "text/html; charset=utf-8")
|> send_resp(200, content)
end
get "/chats/:id" do
chat_record = Repo.get(Chat, id)
tpl_args = [
chat: chat_record,
messages: chat_record |> Message.listing() |> Repo.all()
]
content = render_template("chat", tpl_args)
conn
|> put_resp_header("content-type", "text/html; charset=utf-8")
|> send_resp(200, content)
end
get "/static/*path" do get "/static/*path" do
file_path = Application.app_dir(:bdfr_browser, Path.join("priv/static", path)) file_path = Application.app_dir(:bdfr_browser, Path.join("priv/static", path))

View file

@ -3,7 +3,7 @@ defmodule BdfrBrowser.Importer do
use GenServer use GenServer
alias BdfrBrowser.{Comment, Post, Repo, Subreddit} alias BdfrBrowser.{Chat, Comment, Message, Post, Repo, Subreddit}
def start_link([]) do def start_link([]) do
GenServer.start_link(__MODULE__, [], name: __MODULE__) GenServer.start_link(__MODULE__, [], name: __MODULE__)
@ -49,6 +49,22 @@ defmodule BdfrBrowser.Importer do
List.flatten(result) List.flatten(result)
end end
def chats do
_ = Logger.info("Importing chats ...")
result =
for chat <- read_chats(directory_key: :chat_directory) do
_ = Logger.info("Importing chat `#{chat["id"]}' ...")
{:ok, chat_record} = import_chat(chat)
message_records = for message <- chat["messages"], do: import_message(message, chat_record)
{chat_record, List.flatten(message_records)}
end
List.flatten(result)
end
def background_import do def background_import do
GenServer.cast(__MODULE__, :background_import) GenServer.cast(__MODULE__, :background_import)
end end
@ -64,6 +80,7 @@ defmodule BdfrBrowser.Importer do
def handle_cast(:background_import, state) do def handle_cast(:background_import, state) do
_ = subreddits() _ = subreddits()
_ = posts_and_comments() _ = posts_and_comments()
_ = chats()
{:noreply, state} {:noreply, state}
end end
@ -73,7 +90,8 @@ defmodule BdfrBrowser.Importer do
paths = Keyword.get(args, :paths, []) paths = Keyword.get(args, :paths, [])
extname = Keyword.get(args, :ext, "") extname = Keyword.get(args, :ext, "")
sort = Keyword.get(args, :sort, :desc) sort = Keyword.get(args, :sort, :desc)
base_directory = Application.fetch_env!(:bdfr_browser, :base_directory) directory_key = Keyword.get(args, :directory_key, :base_directory)
base_directory = Application.fetch_env!(:bdfr_browser, directory_key)
[base_directory | paths] [base_directory | paths]
|> Path.join() |> Path.join()
@ -99,6 +117,45 @@ defmodule BdfrBrowser.Importer do
Enum.sort_by(parsed_posts, fn p -> p["created_utc"] end, sort) Enum.sort_by(parsed_posts, fn p -> p["created_utc"] end, sort)
end end
defp read_chats(args) do
directory_key = Keyword.get(args, :directory_key, :chat_directory)
base_directory = Application.fetch_env!(:bdfr_browser, directory_key)
new_chats =
for chat <- list_folders([{:ext, ".json"} | args]) do
file_path = Path.join([base_directory, chat])
parsed = file_path |> File.read!() |> Jason.decode!()
Map.put(parsed, "filename", chat)
end
old_chats =
for chat <- list_folders([{:ext, ".json_lines"} | args]) do
file_path = Path.join([base_directory, chat])
messages =
file_path
|> File.stream!()
|> Stream.map(&String.trim/1)
|> Stream.map(fn line ->
{:ok, [author, date, message]} = Jason.decode(line)
formatted_date = date |> String.replace(" UTC", "Z") |> String.replace(" ", "T")
%{
"author" => author,
"timestamp" => formatted_date,
"content" => %{
"Message" => message
}
}
end)
|> Enum.to_list()
%{"id" => Path.basename(chat, ".json_lines"), "messages" => messages, "filename" => chat}
end
old_chats ++ new_chats
end
defp import_post(post, subreddit) do defp import_post(post, subreddit) do
id = post["id"] id = post["id"]
@ -142,4 +199,38 @@ defmodule BdfrBrowser.Importer do
[parent] ++ children [parent] ++ children
end end
defp import_chat(chat) do
id = chat["id"]
accounts = for message <- chat["messages"], uniq: true, do: message["author"]
%Chat{
id: id,
accounts: accounts
}
|> Repo.insert(
on_conflict: [set: [id: id]],
conflict_target: :id
)
end
defp import_message(message, chat) do
id = :sha3_256 |> :crypto.hash([chat.id, message["timestamp"]]) |> Base.encode16(case: :lower)
{:ok, posted_at, 0} = DateTime.from_iso8601(message["timestamp"])
{:ok, message} =
%Message{
id: id,
author: message["author"],
message: message["content"]["Message"],
posted_at: posted_at,
chat: chat
}
|> Repo.insert(
on_conflict: [set: [id: id]],
conflict_target: :id
)
message
end
end end

View file

@ -0,0 +1,25 @@
defmodule BdfrBrowser.Message do
use Ecto.Schema
import Ecto.Query, only: [from: 2]
alias BdfrBrowser.Chat
@primary_key {:id, :string, autogenerate: false}
@foreign_key_type :string
schema "messages" do
field :author, :string
field :message, :string
field :posted_at, :utc_datetime
belongs_to :chat, Chat
end
def listing(chat) do
from(m in __MODULE__,
where: m.chat_id == ^chat.id,
order_by: [asc: m.posted_at]
)
end
end

View file

@ -0,0 +1,21 @@
defmodule BdfrBrowser.Repo.Migrations.CreateChats do
use Ecto.Migration
def change do
create table(:chats, primary_key: false) do
add :id, :string, primary_key: true, size: 1024
add :accounts, {:array, :string}
end
create table(:messages, primary_key: false) do
add :id, :string, primary_key: true, size: 256
add :author, :string
add :message, :text
add :posted_at, :utc_datetime
add :chat_id, references(:chats, type: :string)
end
create index("messages", :chat_id)
end
end

View file

@ -0,0 +1,18 @@
<h2>Chats</h2>
<%= for message <- messages do %>
<div class="row" style="margin-bottom: 2px;">
<div class="card">
<div class="card-body" style="padding: 8px;">
<blockquote class="blockquote mb-0" style="font-size: 1rem;">
<%= Earmark.as_html!(message.message) %>
<footer class="blockquote-footer">
<%= message.author %>,
<small><%= DateTime.to_iso8601(message.posted_at) %></small>
</footer>
</blockquote>
</div>
</div>
</div>
<% end %>

View file

@ -0,0 +1,16 @@
<h2>Chats</h2>
<div class="row text-center">
<div class="d-grid gap-2 col-12 mx-auto">
<%= for chat <- chats do %>
<div class="card">
<div class="card-body">
<h5 class="card-title"><a href="/chats/<%= URI.encode(chat.id, &URI.char_unreserved?/1) %>"><%= Enum.join(chat.accounts, ", ") %></a></h5>
<h6 class="card-subtitle mb-2 text-body-secondary">
<%= chat.num_messages %> message(s) - <%= DateTime.to_iso8601(chat.latest_message) %>
</h6>
</div>
</div>
<% end %>
</div>
</div>

View file

@ -2,6 +2,8 @@
<div class="row text-center"> <div class="row text-center">
<div class="d-grid gap-2 col-12 mx-auto"> <div class="d-grid gap-2 col-12 mx-auto">
<a class="btn btn-outline-primary btn-lg" href="/chats" role="button">Chats</a>
<%= for subreddit <- subreddits do %> <%= for subreddit <- subreddits do %>
<a class="btn btn-outline-secondary btn-lg" href="/r/<%= subreddit %>" role="button"><%= subreddit %></a> <a class="btn btn-outline-secondary btn-lg" href="/r/<%= subreddit %>" role="button"><%= subreddit %></a>
<% end %> <% end %>