Compare commits
10 commits
4b4c01bb14
...
be98b4b1c8
Author | SHA1 | Date | |
---|---|---|---|
be98b4b1c8 | |||
953f2c3950 | |||
f23b6904c9 | |||
e00e6b97c3 | |||
4851b9bf7c | |||
e635454b2e | |||
956f356a2c | |||
529161f21c | |||
a74a0185f4 | |||
14dceccfec |
10 changed files with 83 additions and 31 deletions
1
.gitignore
vendored
1
.gitignore
vendored
|
@ -19,6 +19,7 @@ erl_crash.dump
|
||||||
# nix
|
# nix
|
||||||
/.direnv
|
/.direnv
|
||||||
/.elixir_ls
|
/.elixir_ls
|
||||||
|
/.lexical
|
||||||
.pre-commit-config.yaml
|
.pre-commit-config.yaml
|
||||||
|
|
||||||
# Exclude releases
|
# Exclude releases
|
||||||
|
|
|
@ -204,15 +204,23 @@ defmodule BdfrBrowser.HTTP.Plug do
|
||||||
defp post_media(post, args) do
|
defp post_media(post, args) do
|
||||||
base_directory = Application.fetch_env!(:bdfr_browser, :base_directory)
|
base_directory = Application.fetch_env!(:bdfr_browser, :base_directory)
|
||||||
post_dir = Path.join([base_directory | Keyword.fetch!(args, :paths)])
|
post_dir = Path.join([base_directory | Keyword.fetch!(args, :paths)])
|
||||||
post_img = "#{post}*.{jpg,JPG,jpeg,JPEG,png,PNG,gif,GIF}"
|
post_img = "#{post}*.{jpg,JPG,jpeg,JPEG,png,PNG,gif,GIF,webp,WEBP}"
|
||||||
post_vid = "#{post}*.{mp4,MP4}"
|
post_vid = "#{post}*.{mp4,MP4,webm,WEBM}"
|
||||||
|
|
||||||
%{
|
%{
|
||||||
images: [post_dir, post_img] |> Path.join() |> Path.wildcard() |> Enum.map(&media_path/1),
|
images: post_media_for_type(post_dir, post_img),
|
||||||
videos: [post_dir, post_vid] |> Path.join() |> Path.wildcard() |> Enum.map(&media_path/1)
|
videos: post_media_for_type(post_dir, post_vid)
|
||||||
}
|
}
|
||||||
end
|
end
|
||||||
|
|
||||||
|
defp post_media_for_type(post_dir, post_type) do
|
||||||
|
[post_dir, post_type]
|
||||||
|
|> Path.join()
|
||||||
|
|> Path.wildcard()
|
||||||
|
|> Enum.map(&media_path/1)
|
||||||
|
|> Enum.sort()
|
||||||
|
end
|
||||||
|
|
||||||
defp media_path(full_path) do
|
defp media_path(full_path) do
|
||||||
base_directory = Application.fetch_env!(:bdfr_browser, :base_directory)
|
base_directory = Application.fetch_env!(:bdfr_browser, :base_directory)
|
||||||
|
|
||||||
|
@ -232,6 +240,8 @@ defmodule BdfrBrowser.HTTP.Plug do
|
||||||
".png" -> "image/png"
|
".png" -> "image/png"
|
||||||
".gif" -> "image/gif"
|
".gif" -> "image/gif"
|
||||||
".mp4" -> "video/mp4"
|
".mp4" -> "video/mp4"
|
||||||
|
".webp" -> "image/webp"
|
||||||
|
".webm" -> "video/webm"
|
||||||
".js" -> "text/javascript"
|
".js" -> "text/javascript"
|
||||||
".css" -> "text/css"
|
".css" -> "text/css"
|
||||||
end
|
end
|
||||||
|
|
|
@ -14,6 +14,7 @@ defmodule BdfrBrowser.Importer do
|
||||||
field :fs_pid, pid
|
field :fs_pid, pid
|
||||||
field :post_changes, [Path.t()], default: MapSet.new()
|
field :post_changes, [Path.t()], default: MapSet.new()
|
||||||
field :chat_changes, [Path.t()], default: MapSet.new()
|
field :chat_changes, [Path.t()], default: MapSet.new()
|
||||||
|
field :last_import, non_neg_integer()
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
|
@ -37,7 +38,7 @@ defmodule BdfrBrowser.Importer do
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
def posts_and_comments do
|
def posts_and_comments(last_import \\ nil) do
|
||||||
_ = Logger.info("Importing posts and comments ...")
|
_ = Logger.info("Importing posts and comments ...")
|
||||||
|
|
||||||
result =
|
result =
|
||||||
|
@ -49,7 +50,7 @@ defmodule BdfrBrowser.Importer do
|
||||||
for date <- list_folders(paths: [subreddit]) do
|
for date <- list_folders(paths: [subreddit]) do
|
||||||
_ = Logger.debug("Importing entries from `#{subreddit}' on `#{date}' ...")
|
_ = Logger.debug("Importing entries from `#{subreddit}' on `#{date}' ...")
|
||||||
|
|
||||||
for post <- read_posts(paths: [subreddit, date], ext: ".json") do
|
for post <- read_posts(paths: [subreddit, date], ext: ".json", last_import: last_import) do
|
||||||
_ = Logger.debug("Importing `#{post["id"]}' from `#{subreddit}' ...")
|
_ = Logger.debug("Importing `#{post["id"]}' from `#{subreddit}' ...")
|
||||||
|
|
||||||
{:ok, post_record} = import_post(post, subreddit_record)
|
{:ok, post_record} = import_post(post, subreddit_record)
|
||||||
|
@ -142,11 +143,11 @@ defmodule BdfrBrowser.Importer do
|
||||||
end
|
end
|
||||||
|
|
||||||
@impl true
|
@impl true
|
||||||
def handle_cast(:background_import, state) do
|
def handle_cast(:background_import, %State{last_import: last_import} = state) do
|
||||||
_ = subreddits()
|
_ = subreddits()
|
||||||
_ = posts_and_comments()
|
_ = posts_and_comments(last_import)
|
||||||
_ = chats()
|
_ = chats()
|
||||||
{:noreply, state}
|
{:noreply, %State{state | last_import: System.os_time(:second)}}
|
||||||
end
|
end
|
||||||
|
|
||||||
@impl true
|
@impl true
|
||||||
|
@ -239,6 +240,7 @@ defmodule BdfrBrowser.Importer do
|
||||||
defp read_posts(args) do
|
defp read_posts(args) do
|
||||||
posts = list_folders(args)
|
posts = list_folders(args)
|
||||||
sort = Keyword.get(args, :sort, :desc)
|
sort = Keyword.get(args, :sort, :desc)
|
||||||
|
last_import = Keyword.get(args, :last_import)
|
||||||
|
|
||||||
base_directory = Application.fetch_env!(:bdfr_browser, :base_directory)
|
base_directory = Application.fetch_env!(:bdfr_browser, :base_directory)
|
||||||
post_dir = Path.join([base_directory | Keyword.fetch!(args, :paths)])
|
post_dir = Path.join([base_directory | Keyword.fetch!(args, :paths)])
|
||||||
|
@ -246,11 +248,25 @@ defmodule BdfrBrowser.Importer do
|
||||||
parsed_posts =
|
parsed_posts =
|
||||||
for post <- posts do
|
for post <- posts do
|
||||||
file_path = Path.join([post_dir, post])
|
file_path = Path.join([post_dir, post])
|
||||||
|
|
||||||
|
if is_nil(last_import) do
|
||||||
parsed = file_path |> File.read!() |> Jason.decode!()
|
parsed = file_path |> File.read!() |> Jason.decode!()
|
||||||
Map.put(parsed, "filename", post)
|
Map.put(parsed, "filename", post)
|
||||||
|
else
|
||||||
|
{:ok, info} = File.stat(file_path, time: :posix)
|
||||||
|
|
||||||
|
if info.mtime > last_import do
|
||||||
|
parsed = file_path |> File.read!() |> Jason.decode!()
|
||||||
|
Map.put(parsed, "filename", post)
|
||||||
|
else
|
||||||
|
nil
|
||||||
|
end
|
||||||
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
Enum.sort_by(parsed_posts, fn p -> p["created_utc"] end, sort)
|
parsed_posts
|
||||||
|
|> Enum.reject(&is_nil/1)
|
||||||
|
|> Enum.sort_by(fn p -> p["created_utc"] end, sort)
|
||||||
end
|
end
|
||||||
|
|
||||||
defp read_chats(args) do
|
defp read_chats(args) do
|
||||||
|
|
|
@ -17,6 +17,9 @@ defmodule BdfrBrowser.RenderUtils do
|
||||||
|> Earmark.as_html!()
|
|> Earmark.as_html!()
|
||||||
end
|
end
|
||||||
|
|
||||||
|
def link_to_user(name) when name in ~w([deleted] DELETED), do: name
|
||||||
|
def link_to_user(name), do: "<a href=\"/user/#{name}\">#{name}</a>"
|
||||||
|
|
||||||
# Helper
|
# Helper
|
||||||
|
|
||||||
defp maybe_insert_image(<<"mxc://reddit.com/", filename::binary>> = msg, replacement) do
|
defp maybe_insert_image(<<"mxc://reddit.com/", filename::binary>> = msg, replacement) do
|
||||||
|
|
2
mix.lock
2
mix.lock
|
@ -7,7 +7,7 @@
|
||||||
"earmark": {:hex, :earmark, "1.4.39", "acdb2f02c536471029dbcc509fbd6b94b89f40ad7729fb3f68f4b6944843f01d", [:mix], [{:earmark_parser, "~> 1.4.33", [hex: :earmark_parser, repo: "hexpm", optional: false]}], "hexpm", "156c9d8ec3cbeccdbf26216d8247bdeeacc8c76b4d9eee7554be2f1b623ea440"},
|
"earmark": {:hex, :earmark, "1.4.39", "acdb2f02c536471029dbcc509fbd6b94b89f40ad7729fb3f68f4b6944843f01d", [:mix], [{:earmark_parser, "~> 1.4.33", [hex: :earmark_parser, repo: "hexpm", optional: false]}], "hexpm", "156c9d8ec3cbeccdbf26216d8247bdeeacc8c76b4d9eee7554be2f1b623ea440"},
|
||||||
"earmark_parser": {:hex, :earmark_parser, "1.4.33", "3c3fd9673bb5dcc9edc28dd90f50c87ce506d1f71b70e3de69aa8154bc695d44", [:mix], [], "hexpm", "2d526833729b59b9fdb85785078697c72ac5e5066350663e5be6a1182da61b8f"},
|
"earmark_parser": {:hex, :earmark_parser, "1.4.33", "3c3fd9673bb5dcc9edc28dd90f50c87ce506d1f71b70e3de69aa8154bc695d44", [:mix], [], "hexpm", "2d526833729b59b9fdb85785078697c72ac5e5066350663e5be6a1182da61b8f"},
|
||||||
"ecto": {:hex, :ecto, "3.10.3", "eb2ae2eecd210b4eb8bece1217b297ad4ff824b4384c0e3fdd28aaf96edd6135", [:mix], [{:decimal, "~> 1.6 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "44bec74e2364d491d70f7e42cd0d690922659d329f6465e89feb8a34e8cd3433"},
|
"ecto": {:hex, :ecto, "3.10.3", "eb2ae2eecd210b4eb8bece1217b297ad4ff824b4384c0e3fdd28aaf96edd6135", [:mix], [{:decimal, "~> 1.6 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "44bec74e2364d491d70f7e42cd0d690922659d329f6465e89feb8a34e8cd3433"},
|
||||||
"ecto_sql": {:hex, :ecto_sql, "3.10.1", "6ea6b3036a0b0ca94c2a02613fd9f742614b5cfe494c41af2e6571bb034dd94c", [:mix], [{:db_connection, "~> 2.4.1 or ~> 2.5", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.10.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.6.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.16.0 or ~> 0.17.0 or ~> 1.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:tds, "~> 2.1.1 or ~> 2.2", [hex: :tds, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "f6a25bdbbd695f12c8171eaff0851fa4c8e72eec1e98c7364402dda9ce11c56b"},
|
"ecto_sql": {:hex, :ecto_sql, "3.10.2", "6b98b46534b5c2f8b8b5f03f126e75e2a73c64f3c071149d32987a5378b0fdbd", [:mix], [{:db_connection, "~> 2.4.1 or ~> 2.5", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.10.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.6.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.16.0 or ~> 0.17.0 or ~> 1.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:tds, "~> 2.1.1 or ~> 2.2", [hex: :tds, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "68c018debca57cb9235e3889affdaec7a10616a4e3a80c99fa1d01fdafaa9007"},
|
||||||
"file_system": {:hex, :file_system, "0.2.10", "fb082005a9cd1711c05b5248710f8826b02d7d1784e7c3451f9c1231d4fc162d", [:mix], [], "hexpm", "41195edbfb562a593726eda3b3e8b103a309b733ad25f3d642ba49696bf715dc"},
|
"file_system": {:hex, :file_system, "0.2.10", "fb082005a9cd1711c05b5248710f8826b02d7d1784e7c3451f9c1231d4fc162d", [:mix], [], "hexpm", "41195edbfb562a593726eda3b3e8b103a309b733ad25f3d642ba49696bf715dc"},
|
||||||
"jason": {:hex, :jason, "1.4.1", "af1504e35f629ddcdd6addb3513c3853991f694921b1b9368b0bd32beb9f1b63", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "fbb01ecdfd565b56261302f7e1fcc27c4fb8f32d56eab74db621fc154604a7a1"},
|
"jason": {:hex, :jason, "1.4.1", "af1504e35f629ddcdd6addb3513c3853991f694921b1b9368b0bd32beb9f1b63", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "fbb01ecdfd565b56261302f7e1fcc27c4fb8f32d56eab74db621fc154604a7a1"},
|
||||||
"mime": {:hex, :mime, "2.0.5", "dc34c8efd439abe6ae0343edbb8556f4d63f178594894720607772a041b04b02", [:mix], [], "hexpm", "da0d64a365c45bc9935cc5c8a7fc5e49a0e0f9932a761c55d6c52b142780a05c"},
|
"mime": {:hex, :mime, "2.0.5", "dc34c8efd439abe6ae0343edbb8556f4d63f178594894720607772a041b04b02", [:mix], [], "hexpm", "da0d64a365c45bc9935cc5c8a7fc5e49a0e0f9932a761c55d6c52b142780a05c"},
|
||||||
|
|
4
mix.nix
4
mix.nix
|
@ -114,12 +114,12 @@ let
|
||||||
|
|
||||||
ecto_sql = buildMix rec {
|
ecto_sql = buildMix rec {
|
||||||
name = "ecto_sql";
|
name = "ecto_sql";
|
||||||
version = "3.10.1";
|
version = "3.10.2";
|
||||||
|
|
||||||
src = fetchHex {
|
src = fetchHex {
|
||||||
pkg = "${name}";
|
pkg = "${name}";
|
||||||
version = "${version}";
|
version = "${version}";
|
||||||
sha256 = "0sy5277akp828hvcg60yxhpfgj543y2z1bqy2z414pv9ppdmp8pn";
|
sha256 = "01whmapzs08xzachra73lhb0d8f7mvysz29qbqivjz55pkg1ih38";
|
||||||
};
|
};
|
||||||
|
|
||||||
beamDeps = [ db_connection ecto postgrex telemetry ];
|
beamDeps = [ db_connection ecto postgrex telemetry ];
|
||||||
|
|
|
@ -9,7 +9,7 @@
|
||||||
<% end %>
|
<% end %>
|
||||||
|
|
||||||
<footer class="blockquote-footer">
|
<footer class="blockquote-footer">
|
||||||
<a href="/user/<%= comment.author %>"><%= comment.author %></a>,
|
<%= BdfrBrowser.RenderUtils.link_to_user(comment.author) %>,
|
||||||
<small><%= DateTime.to_iso8601(comment.posted_at) %></small>
|
<small><%= DateTime.to_iso8601(comment.posted_at) %></small>
|
||||||
</footer>
|
</footer>
|
||||||
</blockquote>
|
</blockquote>
|
||||||
|
|
|
@ -13,7 +13,7 @@
|
||||||
<%= BdfrBrowser.RenderUtils.message(message.message) %>
|
<%= BdfrBrowser.RenderUtils.message(message.message) %>
|
||||||
|
|
||||||
<footer class="blockquote-footer">
|
<footer class="blockquote-footer">
|
||||||
<a href="/user/<%= message.author %>"><%= message.author %></a>,
|
<%= BdfrBrowser.RenderUtils.link_to_user(message.author) %>,
|
||||||
<small><%= DateTime.to_iso8601(message.posted_at) %></small>
|
<small><%= DateTime.to_iso8601(message.posted_at) %></small>
|
||||||
|
|
||||||
<%= unless is_nil(message.bookmark) do %>
|
<%= unless is_nil(message.bookmark) do %>
|
||||||
|
@ -28,18 +28,40 @@
|
||||||
|
|
||||||
<script>
|
<script>
|
||||||
const bookmarks = document.querySelectorAll('[data-bookmark]');
|
const bookmarks = document.querySelectorAll('[data-bookmark]');
|
||||||
|
|
||||||
|
if (bookmarks.length > 0) {
|
||||||
const header = document.getElementsByTagName('h2')[0];
|
const header = document.getElementsByTagName('h2')[0];
|
||||||
let container = document.createElement('p');
|
|
||||||
|
const container = document.createElement('div');
|
||||||
|
container.className = 'dropdown';
|
||||||
|
|
||||||
|
const dropdownButton = document.createElement('button');
|
||||||
|
dropdownButton.id = 'bookmarkDropdown';
|
||||||
|
dropdownButton.className = 'btn btn-secondary dropdown-toggle';
|
||||||
|
dropdownButton.setAttribute('type', 'button');
|
||||||
|
dropdownButton.setAttribute('data-bs-toggle', 'dropdown');
|
||||||
|
dropdownButton.setAttribute('aria-expanded', 'false');
|
||||||
|
dropdownButton.innerText = 'Bookmarks';
|
||||||
|
|
||||||
|
container.appendChild(dropdownButton);
|
||||||
|
|
||||||
|
const dropdownMenu = document.createElement('ul');
|
||||||
|
dropdownMenu.className = 'dropdown-menu';
|
||||||
|
|
||||||
for (var i = 0; i < bookmarks.length; i++) {
|
for (var i = 0; i < bookmarks.length; i++) {
|
||||||
|
console.log(bookmarks[i]);
|
||||||
|
let liElement = document.createElement('li');
|
||||||
let bookmarkElement = document.createElement('a');
|
let bookmarkElement = document.createElement('a');
|
||||||
bookmarkElement.href = `#${bookmarks[i].id}`;
|
bookmarkElement.href = `#${bookmarks[i].id}`;
|
||||||
bookmarkElement.innerText = bookmarks[i].dataset.bookmark;
|
bookmarkElement.innerText = bookmarks[i].dataset.bookmark;
|
||||||
bookmarkElement.className = "btn btn-secondary btn-sm";
|
bookmarkElement.className = "dropdown-item";
|
||||||
bookmarkElement.setAttribute('role', 'button');
|
|
||||||
|
|
||||||
container.appendChild(bookmarkElement);
|
liElement.appendChild(bookmarkElement);
|
||||||
|
dropdownMenu.appendChild(liElement);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
container.appendChild(dropdownMenu);
|
||||||
header.after(container);
|
header.after(container);
|
||||||
|
container.after(document.createElement('br'));
|
||||||
|
}
|
||||||
</script>
|
</script>
|
||||||
|
|
|
@ -1,11 +1,11 @@
|
||||||
<h2><%= post.title %></h2>
|
<h2><%= post.title %></h2>
|
||||||
|
|
||||||
<p>
|
<p>
|
||||||
<small><a href="/user/<%= post.author %>"><%= post.author %></a></small>
|
<small><%= BdfrBrowser.RenderUtils.link_to_user(post.author) %></small>
|
||||||
-
|
-
|
||||||
<small><%= DateTime.to_iso8601(post.posted_at) %></small>
|
<small><%= DateTime.to_iso8601(post.posted_at) %></small>
|
||||||
-
|
-
|
||||||
<a href="https://reddit.com<%= post.permalink %>">Open reddit</a>
|
<a href="https://www.reddit.com<%= post.permalink %>" rel="noreferrer">Open reddit</a>
|
||||||
</p>
|
</p>
|
||||||
|
|
||||||
<%= unless is_nil(post.selftext) do %>
|
<%= unless is_nil(post.selftext) do %>
|
||||||
|
|
|
@ -46,7 +46,7 @@
|
||||||
<%= BdfrBrowser.RenderUtils.comment(comment.body) %>
|
<%= BdfrBrowser.RenderUtils.comment(comment.body) %>
|
||||||
|
|
||||||
<footer class="blockquote-footer">
|
<footer class="blockquote-footer">
|
||||||
<a href="/user/<%= comment.author %>"><%= comment.author %></a>,
|
<%= BdfrBrowser.RenderUtils.link_to_user(comment.author) %>,
|
||||||
<a href="/r/<%= comment.subreddit %>/<%= comment.post_date %>/<%= comment.post_id %>"><%= comment.post_title %></a>,
|
<a href="/r/<%= comment.subreddit %>/<%= comment.post_date %>/<%= comment.post_id %>"><%= comment.post_title %></a>,
|
||||||
<a href="/r/<%= comment.subreddit %>/<%= comment.post_date %>/"><%= comment.subreddit %></a>,
|
<a href="/r/<%= comment.subreddit %>/<%= comment.post_date %>/"><%= comment.subreddit %></a>,
|
||||||
<small><%= DateTime.to_iso8601(comment.posted_at) %></small>
|
<small><%= DateTime.to_iso8601(comment.posted_at) %></small>
|
||||||
|
|
Reference in a new issue