Make S3 service configurable (#190)

Author: raulrpearson

.env.sample

@@ -3,7 +3,7 @@ BASE_URL=http://localhost:4000
 # SECURE_COOKIE=false
 
 DATABASE_URL=postgres://claper:claper@db:5432/claper
-SECRET_KEY_BASE=0LZiQBLw4WvqPlz4cz8RsHJlxNiSqM9B48y4ChyJ5v1oA0L/TPIqRjQNdPZN3iEG # Generate with `mix phx.gen.secret` 
+SECRET_KEY_BASE=0LZiQBLw4WvqPlz4cz8RsHJlxNiSqM9B48y4ChyJ5v1oA0L/TPIqRjQNdPZN3iEG # Generate with `mix phx.gen.secret`
 # ⚠️ Don't use this exact value for SECRET_KEY_BASE or someone would be able to sign a cookie with user_id=1 and log in as the admin!
 
 # Storage configuration
@@ -12,10 +12,19 @@ PRESENTATION_STORAGE=local
 PRESENTATION_STORAGE_DIR=/app/uploads
 #MAX_FILE_SIZE_MB=15
 
-#AWS_ACCESS_KEY_ID=xxx
-#AWS_SECRET_ACCESS_KEY=xxx
-#AWS_REGION=eu-west-3
-#AWS_PRES_BUCKET=xxx
+# The standard AWS environment variables
+#S3_ACCESS_KEY_ID=xxx
+#S3_SECRET_ACCESS_KEY=xxx
+#S3_REGION=eu-west-3
+#S3_BUCKET=xxx
+
+# If you're using an alternative S3-compatible service, port optional
+#S3_SCHEME=https://
+#S3_HOST=www.example.com
+#S3_PORT=443
+
+# If the public S3-compatible URL is different from the one used to write data
+#S3_PUBLIC_URL=https://www.example.com
 
 # Mail configuration
 
@@ -50,4 +59,4 @@ MAIL_FROM_NAME=Claper
 # OIDC_SCOPES="openid email profile"
 # OIDC_LOGO_URL=""
 # OIDC_PROPERTY_MAPPINGS="roles:custom_attributes.roles,organization:custom_attributes.organization"
-# OIDC_AUTO_REDIRECT_LOGIN=true
+# OIDC_AUTO_REDIRECT_LOGIN=true

assets/js/app.js

@@ -685,35 +685,7 @@ Hooks.CSVDownloader = {
 // Merge our custom hooks with the existing hooks
 Object.assign(Hooks, CustomHooks);
 
-let Uploaders = {};
-
-Uploaders.S3 = function (entries, onViewError) {
-  entries.forEach((entry) => {
-    let formData = new FormData();
-    let { url, fields } = entry.meta;
-    Object.entries(fields).forEach(([key, val]) => formData.append(key, val));
-    formData.append("file", entry.file);
-    let xhr = new XMLHttpRequest();
-    onViewError(() => xhr.abort());
-    xhr.onload = () =>
-      xhr.status === 204 ? entry.progress(100) : entry.error();
-    xhr.onerror = () => entry.error();
-    xhr.upload.addEventListener("progress", (event) => {
-      if (event.lengthComputable) {
-        let percent = Math.round((event.loaded / event.total) * 100);
-        if (percent < 100) {
-          entry.progress(percent);
-        }
-      }
-    });
-
-    xhr.open("POST", url, true);
-    xhr.send(formData);
-  });
-};
-
 let liveSocket = new LiveSocket("/live", Socket, {
-  uploaders: Uploaders,
   params: {
     _csrf_token: csrfToken,
     tz: Intl.DateTimeFormat().resolvedOptions().timeZone,

charts/claper/templates/_env.yaml

@@ -16,15 +16,15 @@
   value: claper.co
 - name: DATABASE_URL
   value: postgresql://claper:claper@10.0.0.6:6432/claper
-- name: AWS_ACCESS_KEY_ID
+- name: S3_ACCESS_KEY_ID
   value: XXX
-- name: AWS_PRES_BUCKET
+- name: S3_BUCKET
   value: XXX
 - name: POOL_SIZE
   value: "20"
-- name: AWS_REGION
+- name: S3_REGION
   value: eu-west-3
-- name: AWS_SECRET_ACCESS_KEY
+- name: S3_SECRET_ACCESS_KEY
   value: XXX
 - name: PRESENTATION_STORAGE
   value: s3

config/runtime.exs

@@ -86,9 +86,39 @@ smtp_tls = get_var_from_path_or_env(config_dir, "SMTP_TLS", "always")
 smtp_auth = get_var_from_path_or_env(config_dir, "SMTP_AUTH", "always")
 smtp_port = get_int_from_path_or_env(config_dir, "SMTP_PORT", 25)
 
-aws_access_key_id = get_var_from_path_or_env(config_dir, "AWS_ACCESS_KEY_ID", nil)
-aws_secret_access_key = get_var_from_path_or_env(config_dir, "AWS_SECRET_ACCESS_KEY", nil)
-aws_region = get_var_from_path_or_env(config_dir, "AWS_REGION", nil)
+storage = get_var_from_path_or_env(config_dir, "PRESENTATION_STORAGE", "local")
+if storage not in ["local", "s3"], do: raise("Invalid PRESENTATION_STORAGE value #{storage}")
+
+s3_access_key_id = get_var_from_path_or_env(config_dir, "S3_ACCESS_KEY_ID")
+s3_secret_access_key = get_var_from_path_or_env(config_dir, "S3_SECRET_ACCESS_KEY")
+s3_region = get_var_from_path_or_env(config_dir, "S3_REGION")
+s3_bucket = get_var_from_path_or_env(config_dir, "S3_BUCKET")
+
+if storage == "s3" and
+     not Enum.all?([s3_access_key_id, s3_secret_access_key, s3_region, s3_bucket]) do
+  raise(
+    "S3_ACCESS_KEY_ID, S3_SECRET_ACCESS_KEY, S3_REGION and S3_BUCKET required when PRESENTATION_STORAGE=s3"
+  )
+end
+
+s3_scheme = get_var_from_path_or_env(config_dir, "S3_SCHEME")
+s3_host = get_var_from_path_or_env(config_dir, "S3_HOST")
+s3_port = get_var_from_path_or_env(config_dir, "S3_PORT")
+
+if storage == "s3" do
+  if !!s3_scheme and !s3_host, do: "S3_HOST required if S3_SCHEME is set"
+  if !s3_scheme and !!s3_host, do: "S3_SCHEME required if S3_HOST is set"
+end
+
+s3_public_url =
+  get_var_from_path_or_env(
+    config_dir,
+    "S3_PUBLIC_URL",
+    if(s3_scheme && s3_host,
+      do: s3_scheme <> s3_host <> if(s3_port, do: ":#{s3_port}", else: ""),
+      else: "https://#{s3_bucket}.s3.#{s3_region}.amazonaws.com"
+    )
+  )
 
 same_site_cookie = get_var_from_path_or_env(config_dir, "SAME_SITE_COOKIE", "Lax")
 
@@ -176,9 +206,10 @@ config :claper,
 
 config :claper, :presentations,
   max_file_size: max_file_size,
-  storage: get_var_from_path_or_env(config_dir, "PRESENTATION_STORAGE", "local"),
-  aws_bucket: get_var_from_path_or_env(config_dir, "AWS_PRES_BUCKET", nil),
-  resolution: get_var_from_path_or_env(config_dir, "GS_JPG_RESOLUTION", "300x300")
+  storage: storage,
+  s3_bucket: s3_bucket,
+  resolution: get_var_from_path_or_env(config_dir, "GS_JPG_RESOLUTION", "300x300"),
+  s3_public_url: s3_public_url
 
 config :claper, :mail,
   from: get_var_from_path_or_env(config_dir, "MAIL_FROM", "noreply@claper.co"),
@@ -227,9 +258,10 @@ case mail_transport do
 end
 
 config :ex_aws,
-  access_key_id: aws_access_key_id,
-  secret_access_key: aws_secret_access_key,
-  region: aws_region,
-  normalize_path: false
+  access_key_id: s3_access_key_id,
+  secret_access_key: s3_secret_access_key,
+  region: s3_region,
+  normalize_path: false,
+  s3: [scheme: s3_scheme, host: s3_host, port: s3_port]
 
 config :swoosh, :api_client, Swoosh.ApiClient.Finch

lib/claper/presentations.ex

@@ -31,6 +31,65 @@ defmodule Claper.Presentations do
   def get_presentation_files_by_hash(hash) when is_nil(hash),
     do: []
 
+  @doc """
+  Returns a list of JPG slide URLs for a given presentation.
+
+  When a `Claper.Presentations.PresentationFile{}` struct is provided, the
+  function builds the list of URLs programmatically from the `hash` and
+  `length` fields.
+
+  When an integer or binary `hash` is provided, it queries the database for the
+  associated presentation file and builds the list of URLs programmatically
+  from that.
+
+  When `nil` is provided or when no presentation file is found for the given
+  `hash`, it returns an empty list.
+  """
+  def get_slide_urls(hash_or_presentation_file)
+
+  def get_slide_urls(nil), do: []
+
+  def get_slide_urls(hash) when is_integer(hash), do: get_slide_urls(to_string(hash))
+
+  def get_slide_urls(hash) when is_binary(hash) do
+    case Repo.get_by(PresentationFile, hash: hash) do
+      nil ->
+        []
+
+      presentation ->
+        get_slide_urls(hash, presentation.length)
+    end
+  end
+
+  def get_slide_urls(%PresentationFile{} = presentation) do
+    get_slide_urls(presentation.hash, presentation.length)
+  end
+
+  @doc """
+  Returns a list of JPG slide URLs for a given presentation `hash` and
+  `length`. See also `get_slide_urls/1`.
+  """
+  def get_slide_urls(hash, length) when is_binary(hash) and is_integer(length) do
+    config = Application.get_env(:claper, :presentations)
+
+    case Keyword.fetch!(config, :storage) do
+      "local" ->
+        for index <- 1..length do
+          "/uploads/#{hash}/#{index}.jpg"
+        end
+
+      "s3" ->
+        base_url = Keyword.fetch!(config, :s3_public_url)
+
+        for index <- 1..length do
+          base_url <> "/presentations/#{hash}/#{index}.jpg"
+        end
+
+      storage ->
+        raise "Unrecognised presentations storage value #{storage}"
+    end
+  end
+
   @doc """
   Creates a presentation_files.
 

lib/claper/tasks/converter.ex

@@ -5,7 +5,6 @@ defmodule Claper.Tasks.Converter do
   """
 
   alias Claper.Events
-  alias ExAws.S3
   alias Porcelain.Result
 
   @doc """
@@ -50,11 +49,11 @@ defmodule Claper.Tasks.Converter do
       )
     else
       stream =
-        ExAws.S3.list_objects(get_aws_bucket(), prefix: "presentations/#{hash}")
+        ExAws.S3.list_objects(get_s3_bucket(), prefix: "presentations/#{hash}")
         |> ExAws.stream!()
         |> Stream.map(& &1.key)
 
-      ExAws.S3.delete_all_objects(get_aws_bucket(), stream) |> ExAws.request()
+      ExAws.S3.delete_all_objects(get_s3_bucket(), stream) |> ExAws.request()
     end
   end
 
@@ -134,9 +133,9 @@ defmodule Claper.Tasks.Converter do
         IO.puts("Uploads #{f} to presentations/#{new_hash}/#{Path.basename(f)}")
 
         f
-        |> S3.Upload.stream_file()
-        |> S3.upload(
-          get_aws_bucket(),
+        |> ExAws.S3.Upload.stream_file()
+        |> ExAws.S3.upload(
+          get_s3_bucket(),
           "presentations/#{new_hash}/#{Path.basename(f)}",
           acl: "public-read"
         )
@@ -187,8 +186,8 @@ defmodule Claper.Tasks.Converter do
     Application.get_env(:claper, :storage_dir)
   end
 
-  defp get_aws_bucket do
-    Application.get_env(:claper, :presentations) |> Keyword.get(:aws_bucket)
+  defp get_s3_bucket do
+    Application.get_env(:claper, :presentations) |> Keyword.get(:s3_bucket)
   end
 
   defp get_resolution do

lib/claper_web/live/event_live/manage.ex

@@ -1,11 +1,8 @@
 defmodule ClaperWeb.EventLive.Manage do
   use ClaperWeb, :live_view
 
+  alias Claper.{Embeds, Forms, Polls, Presentations, Quizzes}
   alias ClaperWeb.Presence
-  alias Claper.Polls
-  alias Claper.Forms
-  alias Claper.Embeds
-  alias Claper.Quizzes
 
   @impl true
   def mount(%{"code" => code}, session, socket) do

lib/claper_web/live/event_live/manage.html.heex

@@ -669,30 +669,22 @@
         id="slides-layout"
         class="flex overflow-x-auto w-full md:h-full"
       >
-        <%= for index <- 0..max(0, @event.presentation_file.length-1) do %>
-          <button
-            id={"slide-preview-#{index}"}
-            phx-click="current-page"
-            phx-value-page={index}
-            class="h-full w-full contents"
-          >
-            <%= if Application.get_env(:claper, :presentations) |> Keyword.get(:storage) == "local" do %>
-              <img
-                class={"#{if @state.position==index, do: "border-4 border-primary-500" , else: "opacity-20" }
-                    transition-all object-contain"}
-                src={"/uploads/#{@event.presentation_file.hash}/#{index+1}.jpg"}
-              />
-            <% else %>
-              <img
-                class={"#{if @state.position==index, do: "border-4 border-primary-500" , else: "opacity-20" }
+        <button
+          :for={
+            {src, index} <-
+              Presentations.get_slide_urls(@event.presentation_file) |> Enum.with_index(0)
+          }
+          id={"slide-preview-#{index}"}
+          phx-click="current-page"
+          phx-value-page={index}
+          class="h-full w-full contents"
+        >
+          <img
+            src={src}
+            class={"#{if @state.position==index, do: "border-4 border-primary-500" , else: "opacity-20" }
                     transition-all object-contain"}
-                src={"https://#{Application.get_env(:claper, :presentations) |>
-                          Keyword.get(:aws_bucket)}.s3.#{Application.get_env(:ex_aws,
-                          :region)}.amazonaws.com/presentations/#{@event.presentation_file.hash}/#{index+1}.jpg"}
-              />
-            <% end %>
-          </button>
-        <% end %>
+          />
+        </button>
       </div>
       <div
         :if={@event.presentation_file.length > 0}

lib/claper_web/live/event_live/presenter.ex

@@ -6,6 +6,8 @@ defmodule ClaperWeb.EventLive.Presenter do
   alias Claper.Polls.Poll
   alias Claper.Forms.Form
   alias Claper.Quizzes.Quiz
+  alias Claper.Presentations
+
   @impl true
   def mount(%{"code" => code} = params, session, socket) do
     with %{"locale" => locale} <- session do

lib/claper_web/live/event_live/presenter.html.heex

@@ -216,21 +216,11 @@
         </div>
       <% end %>
       <div class={"#{if @current_embed, do: "hidden", else: ""} text-center"} id="slider">
-        <%= for index <- 1..max(1, @event.presentation_file.length) do %>
-          <%= if @event.presentation_file.length > 0 do %>
-            <%= if Application.get_env(:claper, :presentations) |> Keyword.get(:storage) == "local" do %>
-              <img
-                class="max-h-screen w-auto!"
-                src={"/uploads/#{@event.presentation_file.hash}/#{index}.jpg"}
-              />
-            <% else %>
-              <img
-                class=" max-h-screen w-auto!"
-                src={"https://#{Application.get_env(:claper, :presentations) |> Keyword.get(:aws_bucket)}.s3.#{Application.get_env(:ex_aws, :region)}.amazonaws.com/presentations/#{@event.presentation_file.hash}/#{index}.jpg"}
-              />
-            <% end %>
-          <% end %>
-        <% end %>
+        <img
+          :for={src <- Presentations.get_slide_urls(@event.presentation_file)}
+          src={src}
+          class="max-h-screen w-auto!"
+        />
       </div>
     </div>
   </div>