feat: Support importing and viewing chats
This commit is contained in:
parent
82e435ddd8
commit
23a1d6a9d6
9 changed files with 225 additions and 3 deletions
|
@ -2,6 +2,7 @@ import Config
|
||||||
|
|
||||||
config :bdfr_browser,
|
config :bdfr_browser,
|
||||||
base_directory: System.get_env("BDFR_BROWSER_BASE_DIRECTORY", "/nonexistant"),
|
base_directory: System.get_env("BDFR_BROWSER_BASE_DIRECTORY", "/nonexistant"),
|
||||||
|
chat_directory: System.get_env("BDFR_BROWSER_CHAT_DIRECTORY", "/nonexistant"),
|
||||||
http_ip: to_charlist(System.get_env("BDFR_BROWSER_HTTP_IP", "127.0.0.1")),
|
http_ip: to_charlist(System.get_env("BDFR_BROWSER_HTTP_IP", "127.0.0.1")),
|
||||||
http_port: String.to_integer(System.get_env("BDFR_BROWSER_HTTP_PORT", "4040"))
|
http_port: String.to_integer(System.get_env("BDFR_BROWSER_HTTP_PORT", "4040"))
|
||||||
|
|
||||||
|
|
24
lib/bdfr_browser/chat.ex
Normal file
24
lib/bdfr_browser/chat.ex
Normal file
|
@ -0,0 +1,24 @@
|
||||||
|
defmodule BdfrBrowser.Chat do
|
||||||
|
use Ecto.Schema
|
||||||
|
|
||||||
|
import Ecto.Query, only: [from: 2]
|
||||||
|
|
||||||
|
alias BdfrBrowser.Message
|
||||||
|
|
||||||
|
@primary_key {:id, :string, autogenerate: false}
|
||||||
|
|
||||||
|
schema "chats" do
|
||||||
|
field :accounts, {:array, :string}
|
||||||
|
|
||||||
|
has_many :messages, Message
|
||||||
|
end
|
||||||
|
|
||||||
|
def listing do
|
||||||
|
from(c in __MODULE__,
|
||||||
|
left_join: m in assoc(c, :messages),
|
||||||
|
select: %{id: c.id, accounts: c.accounts, num_messages: count(m.id), latest_message: max(m.posted_at)},
|
||||||
|
order_by: [desc: max(m.posted_at)],
|
||||||
|
group_by: c.id
|
||||||
|
)
|
||||||
|
end
|
||||||
|
end
|
|
@ -1,7 +1,7 @@
|
||||||
defmodule BdfrBrowser.HTTP.Plug do
|
defmodule BdfrBrowser.HTTP.Plug do
|
||||||
use Plug.Router
|
use Plug.Router
|
||||||
|
|
||||||
alias BdfrBrowser.{Repo, Post, Subreddit}
|
alias BdfrBrowser.{Chat, Message, Repo, Post, Subreddit}
|
||||||
|
|
||||||
plug :match
|
plug :match
|
||||||
plug :dispatch
|
plug :dispatch
|
||||||
|
@ -64,6 +64,30 @@ defmodule BdfrBrowser.HTTP.Plug do
|
||||||
|> send_resp(200, content)
|
|> send_resp(200, content)
|
||||||
end
|
end
|
||||||
|
|
||||||
|
get "/chats" do
|
||||||
|
tpl_args = [chats: Chat.listing() |> Repo.all()]
|
||||||
|
content = render_template("chats", tpl_args)
|
||||||
|
|
||||||
|
conn
|
||||||
|
|> put_resp_header("content-type", "text/html; charset=utf-8")
|
||||||
|
|> send_resp(200, content)
|
||||||
|
end
|
||||||
|
|
||||||
|
get "/chats/:id" do
|
||||||
|
chat_record = Repo.get(Chat, id)
|
||||||
|
|
||||||
|
tpl_args = [
|
||||||
|
chat: chat_record,
|
||||||
|
messages: chat_record |> Message.listing() |> Repo.all()
|
||||||
|
]
|
||||||
|
|
||||||
|
content = render_template("chat", tpl_args)
|
||||||
|
|
||||||
|
conn
|
||||||
|
|> put_resp_header("content-type", "text/html; charset=utf-8")
|
||||||
|
|> send_resp(200, content)
|
||||||
|
end
|
||||||
|
|
||||||
get "/static/*path" do
|
get "/static/*path" do
|
||||||
file_path = Application.app_dir(:bdfr_browser, Path.join("priv/static", path))
|
file_path = Application.app_dir(:bdfr_browser, Path.join("priv/static", path))
|
||||||
|
|
||||||
|
|
|
@ -3,7 +3,7 @@ defmodule BdfrBrowser.Importer do
|
||||||
|
|
||||||
use GenServer
|
use GenServer
|
||||||
|
|
||||||
alias BdfrBrowser.{Comment, Post, Repo, Subreddit}
|
alias BdfrBrowser.{Chat, Comment, Message, Post, Repo, Subreddit}
|
||||||
|
|
||||||
def start_link([]) do
|
def start_link([]) do
|
||||||
GenServer.start_link(__MODULE__, [], name: __MODULE__)
|
GenServer.start_link(__MODULE__, [], name: __MODULE__)
|
||||||
|
@ -49,6 +49,22 @@ defmodule BdfrBrowser.Importer do
|
||||||
List.flatten(result)
|
List.flatten(result)
|
||||||
end
|
end
|
||||||
|
|
||||||
|
def chats do
|
||||||
|
_ = Logger.info("Importing chats ...")
|
||||||
|
|
||||||
|
result =
|
||||||
|
for chat <- read_chats(directory_key: :chat_directory) do
|
||||||
|
_ = Logger.info("Importing chat `#{chat["id"]}' ...")
|
||||||
|
|
||||||
|
{:ok, chat_record} = import_chat(chat)
|
||||||
|
message_records = for message <- chat["messages"], do: import_message(message, chat_record)
|
||||||
|
|
||||||
|
{chat_record, List.flatten(message_records)}
|
||||||
|
end
|
||||||
|
|
||||||
|
List.flatten(result)
|
||||||
|
end
|
||||||
|
|
||||||
def background_import do
|
def background_import do
|
||||||
GenServer.cast(__MODULE__, :background_import)
|
GenServer.cast(__MODULE__, :background_import)
|
||||||
end
|
end
|
||||||
|
@ -64,6 +80,7 @@ defmodule BdfrBrowser.Importer do
|
||||||
def handle_cast(:background_import, state) do
|
def handle_cast(:background_import, state) do
|
||||||
_ = subreddits()
|
_ = subreddits()
|
||||||
_ = posts_and_comments()
|
_ = posts_and_comments()
|
||||||
|
_ = chats()
|
||||||
{:noreply, state}
|
{:noreply, state}
|
||||||
end
|
end
|
||||||
|
|
||||||
|
@ -73,7 +90,8 @@ defmodule BdfrBrowser.Importer do
|
||||||
paths = Keyword.get(args, :paths, [])
|
paths = Keyword.get(args, :paths, [])
|
||||||
extname = Keyword.get(args, :ext, "")
|
extname = Keyword.get(args, :ext, "")
|
||||||
sort = Keyword.get(args, :sort, :desc)
|
sort = Keyword.get(args, :sort, :desc)
|
||||||
base_directory = Application.fetch_env!(:bdfr_browser, :base_directory)
|
directory_key = Keyword.get(args, :directory_key, :base_directory)
|
||||||
|
base_directory = Application.fetch_env!(:bdfr_browser, directory_key)
|
||||||
|
|
||||||
[base_directory | paths]
|
[base_directory | paths]
|
||||||
|> Path.join()
|
|> Path.join()
|
||||||
|
@ -99,6 +117,45 @@ defmodule BdfrBrowser.Importer do
|
||||||
Enum.sort_by(parsed_posts, fn p -> p["created_utc"] end, sort)
|
Enum.sort_by(parsed_posts, fn p -> p["created_utc"] end, sort)
|
||||||
end
|
end
|
||||||
|
|
||||||
|
defp read_chats(args) do
|
||||||
|
directory_key = Keyword.get(args, :directory_key, :chat_directory)
|
||||||
|
base_directory = Application.fetch_env!(:bdfr_browser, directory_key)
|
||||||
|
|
||||||
|
new_chats =
|
||||||
|
for chat <- list_folders([{:ext, ".json"} | args]) do
|
||||||
|
file_path = Path.join([base_directory, chat])
|
||||||
|
parsed = file_path |> File.read!() |> Jason.decode!()
|
||||||
|
Map.put(parsed, "filename", chat)
|
||||||
|
end
|
||||||
|
|
||||||
|
old_chats =
|
||||||
|
for chat <- list_folders([{:ext, ".json_lines"} | args]) do
|
||||||
|
file_path = Path.join([base_directory, chat])
|
||||||
|
|
||||||
|
messages =
|
||||||
|
file_path
|
||||||
|
|> File.stream!()
|
||||||
|
|> Stream.map(&String.trim/1)
|
||||||
|
|> Stream.map(fn line ->
|
||||||
|
{:ok, [author, date, message]} = Jason.decode(line)
|
||||||
|
formatted_date = date |> String.replace(" UTC", "Z") |> String.replace(" ", "T")
|
||||||
|
|
||||||
|
%{
|
||||||
|
"author" => author,
|
||||||
|
"timestamp" => formatted_date,
|
||||||
|
"content" => %{
|
||||||
|
"Message" => message
|
||||||
|
}
|
||||||
|
}
|
||||||
|
end)
|
||||||
|
|> Enum.to_list()
|
||||||
|
|
||||||
|
%{"id" => Path.basename(chat, ".json_lines"), "messages" => messages, "filename" => chat}
|
||||||
|
end
|
||||||
|
|
||||||
|
old_chats ++ new_chats
|
||||||
|
end
|
||||||
|
|
||||||
defp import_post(post, subreddit) do
|
defp import_post(post, subreddit) do
|
||||||
id = post["id"]
|
id = post["id"]
|
||||||
|
|
||||||
|
@ -142,4 +199,38 @@ defmodule BdfrBrowser.Importer do
|
||||||
|
|
||||||
[parent] ++ children
|
[parent] ++ children
|
||||||
end
|
end
|
||||||
|
|
||||||
|
defp import_chat(chat) do
|
||||||
|
id = chat["id"]
|
||||||
|
accounts = for message <- chat["messages"], uniq: true, do: message["author"]
|
||||||
|
|
||||||
|
%Chat{
|
||||||
|
id: id,
|
||||||
|
accounts: accounts
|
||||||
|
}
|
||||||
|
|> Repo.insert(
|
||||||
|
on_conflict: [set: [id: id]],
|
||||||
|
conflict_target: :id
|
||||||
|
)
|
||||||
|
end
|
||||||
|
|
||||||
|
defp import_message(message, chat) do
|
||||||
|
id = :sha3_256 |> :crypto.hash([chat.id, message["timestamp"]]) |> Base.encode16(case: :lower)
|
||||||
|
{:ok, posted_at, 0} = DateTime.from_iso8601(message["timestamp"])
|
||||||
|
|
||||||
|
{:ok, message} =
|
||||||
|
%Message{
|
||||||
|
id: id,
|
||||||
|
author: message["author"],
|
||||||
|
message: message["content"]["Message"],
|
||||||
|
posted_at: posted_at,
|
||||||
|
chat: chat
|
||||||
|
}
|
||||||
|
|> Repo.insert(
|
||||||
|
on_conflict: [set: [id: id]],
|
||||||
|
conflict_target: :id
|
||||||
|
)
|
||||||
|
|
||||||
|
message
|
||||||
|
end
|
||||||
end
|
end
|
||||||
|
|
25
lib/bdfr_browser/message.ex
Normal file
25
lib/bdfr_browser/message.ex
Normal file
|
@ -0,0 +1,25 @@
|
||||||
|
defmodule BdfrBrowser.Message do
|
||||||
|
use Ecto.Schema
|
||||||
|
|
||||||
|
import Ecto.Query, only: [from: 2]
|
||||||
|
|
||||||
|
alias BdfrBrowser.Chat
|
||||||
|
|
||||||
|
@primary_key {:id, :string, autogenerate: false}
|
||||||
|
@foreign_key_type :string
|
||||||
|
|
||||||
|
schema "messages" do
|
||||||
|
field :author, :string
|
||||||
|
field :message, :string
|
||||||
|
field :posted_at, :utc_datetime
|
||||||
|
|
||||||
|
belongs_to :chat, Chat
|
||||||
|
end
|
||||||
|
|
||||||
|
def listing(chat) do
|
||||||
|
from(m in __MODULE__,
|
||||||
|
where: m.chat_id == ^chat.id,
|
||||||
|
order_by: [asc: m.posted_at]
|
||||||
|
)
|
||||||
|
end
|
||||||
|
end
|
21
priv/repo/migrations/20230814110852_create_chats.exs
Normal file
21
priv/repo/migrations/20230814110852_create_chats.exs
Normal file
|
@ -0,0 +1,21 @@
|
||||||
|
defmodule BdfrBrowser.Repo.Migrations.CreateChats do
|
||||||
|
use Ecto.Migration
|
||||||
|
|
||||||
|
def change do
|
||||||
|
create table(:chats, primary_key: false) do
|
||||||
|
add :id, :string, primary_key: true, size: 1024
|
||||||
|
add :accounts, {:array, :string}
|
||||||
|
end
|
||||||
|
|
||||||
|
create table(:messages, primary_key: false) do
|
||||||
|
add :id, :string, primary_key: true, size: 256
|
||||||
|
add :author, :string
|
||||||
|
add :message, :text
|
||||||
|
add :posted_at, :utc_datetime
|
||||||
|
|
||||||
|
add :chat_id, references(:chats, type: :string)
|
||||||
|
end
|
||||||
|
|
||||||
|
create index("messages", :chat_id)
|
||||||
|
end
|
||||||
|
end
|
18
priv/templates/http/chat.eex
Normal file
18
priv/templates/http/chat.eex
Normal file
|
@ -0,0 +1,18 @@
|
||||||
|
<h2>Chats</h2>
|
||||||
|
|
||||||
|
<%= for message <- messages do %>
|
||||||
|
<div class="row" style="margin-bottom: 2px;">
|
||||||
|
<div class="card">
|
||||||
|
<div class="card-body" style="padding: 8px;">
|
||||||
|
<blockquote class="blockquote mb-0" style="font-size: 1rem;">
|
||||||
|
<%= Earmark.as_html!(message.message) %>
|
||||||
|
|
||||||
|
<footer class="blockquote-footer">
|
||||||
|
<%= message.author %>,
|
||||||
|
<small><%= DateTime.to_iso8601(message.posted_at) %></small>
|
||||||
|
</footer>
|
||||||
|
</blockquote>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<% end %>
|
16
priv/templates/http/chats.eex
Normal file
16
priv/templates/http/chats.eex
Normal file
|
@ -0,0 +1,16 @@
|
||||||
|
<h2>Chats</h2>
|
||||||
|
|
||||||
|
<div class="row text-center">
|
||||||
|
<div class="d-grid gap-2 col-12 mx-auto">
|
||||||
|
<%= for chat <- chats do %>
|
||||||
|
<div class="card">
|
||||||
|
<div class="card-body">
|
||||||
|
<h5 class="card-title"><a href="/chats/<%= URI.encode(chat.id, &URI.char_unreserved?/1) %>"><%= Enum.join(chat.accounts, ", ") %></a></h5>
|
||||||
|
<h6 class="card-subtitle mb-2 text-body-secondary">
|
||||||
|
<%= chat.num_messages %> message(s) - <%= DateTime.to_iso8601(chat.latest_message) %>
|
||||||
|
</h6>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<% end %>
|
||||||
|
</div>
|
||||||
|
</div>
|
|
@ -2,6 +2,8 @@
|
||||||
|
|
||||||
<div class="row text-center">
|
<div class="row text-center">
|
||||||
<div class="d-grid gap-2 col-12 mx-auto">
|
<div class="d-grid gap-2 col-12 mx-auto">
|
||||||
|
<a class="btn btn-outline-primary btn-lg" href="/chats" role="button">Chats</a>
|
||||||
|
|
||||||
<%= for subreddit <- subreddits do %>
|
<%= for subreddit <- subreddits do %>
|
||||||
<a class="btn btn-outline-secondary btn-lg" href="/r/<%= subreddit %>" role="button"><%= subreddit %></a>
|
<a class="btn btn-outline-secondary btn-lg" href="/r/<%= subreddit %>" role="button"><%= subreddit %></a>
|
||||||
<% end %>
|
<% end %>
|
||||||
|
|
Reference in a new issue