From 6b1729f4a7c8607fcd115be5a48fbba9865d0eed Mon Sep 17 00:00:00 2001 From: Mikko Ahlroth Date: Tue, 21 Mar 2017 07:44:44 +0200 Subject: [PATCH] Initial commit --- .gitignore | 20 +++ LICENSE | 27 +++ README.md | 18 ++ config/config.exs | 30 ++++ lib/apprunner.exs | 86 +++++++++ lib/buildtask.ex | 44 +++++ lib/utils.ex | 407 +++++++++++++++++++++++++++++++++++++++++++ mix.exs | 36 ++++ mix.lock | 3 + test/fbu_test.exs | 77 ++++++++ test/test_helper.exs | 1 + 11 files changed, 749 insertions(+) create mode 100644 .gitignore create mode 100644 LICENSE create mode 100644 README.md create mode 100644 config/config.exs create mode 100644 lib/apprunner.exs create mode 100644 lib/buildtask.ex create mode 100644 lib/utils.ex create mode 100644 mix.exs create mode 100644 mix.lock create mode 100644 test/fbu_test.exs create mode 100644 test/test_helper.exs diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..b6012c7 --- /dev/null +++ b/.gitignore @@ -0,0 +1,20 @@ +# The directory Mix will write compiled artifacts to. +/_build + +# If you run "mix test --cover", coverage assets end up here. +/cover + +# The directory Mix downloads your dependencies sources to. +/deps + +# Where 3rd-party dependencies like ExDoc output generated docs. +/doc + +# Ignore .fetch files in case you like to edit your project deps locally. +/.fetch + +# If the VM crashes, it generates a dump, let's ignore it too. +erl_crash.dump + +# Also ignore archive artifacts (built via "mix archive.build"). +*.ez diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..48fb839 --- /dev/null +++ b/LICENSE @@ -0,0 +1,27 @@ +Copyright © 2017, Mikko Ahlroth +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this +list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation +and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its contributors +may be used to endorse or promote products derived from this software without +specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..d6bb7ff --- /dev/null +++ b/README.md @@ -0,0 +1,18 @@ +# FBU: Frontend Build Utilities + +**TODO: Add description** + +## Installation + +If [available in Hex](https://hex.pm/docs/publish), the package can be installed +by adding `fbu` to your list of dependencies in `mix.exs`: + +```elixir +def deps do + [{:fbu, "~> 0.1.0"}] +end +``` + +Documentation can be generated with [ExDoc](https://github.com/elixir-lang/ex_doc) +and published on [HexDocs](https://hexdocs.pm). Once published, the docs can +be found at [https://hexdocs.pm/fbu](https://hexdocs.pm/fbu). diff --git a/config/config.exs b/config/config.exs new file mode 100644 index 0000000..5de25f0 --- /dev/null +++ b/config/config.exs @@ -0,0 +1,30 @@ +# This file is responsible for configuring your application +# and its dependencies with the aid of the Mix.Config module. +use Mix.Config + +# This configuration is loaded before any dependency and is restricted +# to this project. If another project depends on this project, this +# file won't be loaded nor affect the parent project. For this reason, +# if you want to provide default values for your application for +# 3rd-party users, it should be done in your "mix.exs" file. + +# You can configure for your application as: +# +# config :fbu, key: :value +# +# And access this configuration in your application as: +# +# Application.get_env(:fbu, :key) +# +# Or configure a 3rd-party app: +# +# config :logger, level: :info +# + +# It is also possible to import configuration files, relative to this +# directory. For example, you can emulate configuration per environment +# by uncommenting the line below and defining dev.exs, test.exs and such. +# Configuration from the imported file will override the ones defined +# here (which is why it is important to import them last). +# +# import_config "#{Mix.env}.exs" diff --git a/lib/apprunner.exs b/lib/apprunner.exs new file mode 100644 index 0000000..5b5cc38 --- /dev/null +++ b/lib/apprunner.exs @@ -0,0 +1,86 @@ +defmodule FBU.AppRunner do + @moduledoc """ + AppRunner is a script for ensuring that external apps are terminated + properly when executed from BEAM. + + If you start an application that does not listen to STDIN from BEAM + and BEAM is terminated, the application's STDIN is closed but it may + never close. AppRunner is meant to sit in between, monitoring the + STDIN pipe and killing the application if STDIN closes. This ensures + no zombie processes are left behind if the host BEAM dies. + + Output from the application will be sent back to the host BEAM. If the + application dies for any reason, AppRunner is also closed. + """ + + def wait_for_eof() do + case IO.getn("", 1024) do + :eof -> nil + _ -> wait_for_eof() + end + end + + def exec(program, args) do + Port.open( + {:spawn_executable, program}, + [ + :exit_status, + :stderr_to_stdout, # Redirect stderr to stdout to log properly + args: args, + line: 1024 + ] + ) + end + + def get_pid(port) do + case Port.info(port) do + nil -> + nil + + info when is_list(info) -> + case Keyword.get(info, :os_pid) do + nil -> nil + pid -> Integer.to_string(pid) + end + end + end + + def kill(pid) do + System.find_executable("kill") |> System.cmd([pid]) + end + + def wait_loop(port) do + receive do + {_, {:data, {:eol, msg}}} -> + msg |> :unicode.characters_to_binary(:unicode) |> IO.puts() + + {_, {:data, {:noeol, msg}}} -> + msg |> :unicode.characters_to_binary(:unicode) |> IO.write() + + {_, :eof_received} -> + get_pid(port) |> kill() + :erlang.halt(0) + + {_, :closed} -> + :erlang.halt(0) + + {_, {:exit_status, status}} -> + :erlang.halt(status) + + {:EXIT, _, _} -> + :erlang.halt(1) + end + + wait_loop(port) + end +end + +[program | args] = System.argv() +port = FBU.AppRunner.exec(program, args) + +Task.async(fn -> + FBU.AppRunner.wait_for_eof() + :eof_received +end) + +FBU.AppRunner.wait_loop(port) diff --git a/lib/buildtask.ex b/lib/buildtask.ex new file mode 100644 index 0000000..0f4a398 --- /dev/null +++ b/lib/buildtask.ex @@ -0,0 +1,44 @@ +defmodule FBU.BuildTask do + @moduledoc """ + BuildTask contains the macros that are used for making the tasks look nicer. + """ + + @doc """ + Sets up the necessary things for a build task. + + Each build task should have `use FBU.BuildTask` at its beginning. BuildTask automatically + requires Logger so that doesn't need to be added into the task itself. + """ + defmacro __using__(_opts) do + quote do + use Mix.Task + require Logger + + import FBU.BuildTask + + # Dependencies of the task that will be automatically run before it + @deps [] + end + end + + @doc """ + Replacement for Mix.Task's run/1, used similarly. Code inside will be run when the task + is called, with @deps being run first unless `deps: false` is given in the arguments. + """ + defmacro task(args, do: block) do + quote do + def run(unquote(args) = fbu_buildtask_args) do + task = Mix.Task.task_name(__MODULE__) + Logger.info("[Started] #{task}") + + if Keyword.get(fbu_buildtask_args, :deps, true) and not Enum.empty?(@deps) do + FBU.TaskUtils.run_tasks(@deps) + end + + unquote(block) + + Logger.info("[Finished] #{task}") + end + end + end +end diff --git a/lib/utils.ex b/lib/utils.ex new file mode 100644 index 0000000..0992f97 --- /dev/null +++ b/lib/utils.ex @@ -0,0 +1,407 @@ +defmodule FBU.TaskUtils do + @moduledoc """ + Utilities for project build tasks. + """ + + require Logger + + @elixir System.find_executable("elixir") + + @default_task_timeout 60000 + + defmodule ProgramSpec do + @moduledoc """ + Program that is executed with arguments. Name is used for prefixing logs. + """ + defstruct [ + name: "", + port: nil, + pending_output: "" + ] + end + + defmodule WatchSpec do + @moduledoc """ + Watch specification, target to watch and callback to execute on events. + Name is used for prefixing logs. + + Callback must have arity 2, gets filename/path and list of events as arguments. + """ + defstruct [ + name: "", + path: "", + callback: nil, + pid: nil, + name_atom: nil + ] + end + + @doc """ + Get configuration value. + """ + def conf(key) when is_atom(key) do + Application.get_env(:code_stats, key) + end + + @doc """ + Get absolute path to a program in $PATH. + """ + def exec_path(program) do + System.find_executable(program) + end + + @doc """ + Run the given Mix task and wait for it to stop before returning. + + See run_tasks/2 for the argument description. + """ + def run_task(task, args \\ []) do + run_tasks([{task, args}]) + end + + @doc """ + Run the given tasks in parallel and wait for them all to stop + before returning. + + Valid task types: + + * `{task_name, args}`, where `task_name` is a Mix task name or module, + * `task_name`, where `task_name` is a Mix task name or module to call without + arguments, or + * `task_function` where `task_function` is a function to call without arguments. + """ + def run_tasks(tasks) do + tasks + + |> Enum.map(fn + task_name when is_atom(task_name) or is_binary(task_name) -> + {task_name, []} + + {task_name, args} when is_atom(task_name) or is_binary(task_name) -> + {task_name, args} + + fun when is_function(fun) -> + fun + end) + + # Convert module references to string task names + |> Enum.map(fn + {task_module, args} when is_atom(task_module) -> + {Mix.Task.task_name(task_module), args} + + {task_name, args} when is_binary(task_name) -> + {task_name, args} + + fun when is_function(fun) -> + fun + end) + + |> Enum.map(fn + {task, args} -> + # Rerun is used here to enable tasks being run again in watches + fn -> Mix.Task.rerun(task, args) end + + fun when is_function(fun) -> + fun + end) + + |> run_funs(@default_task_timeout) + end + + @doc """ + Run the given functions in parallel and wait for them all to stop + before returning. + + Functions can either be anonymous functions or tuples of + {module, fun, args}. + """ + def run_funs(funs, timeout \\ @default_task_timeout) when is_list(funs) do + funs + |> Enum.map(fn + fun when is_function(fun) -> + Task.async(fun) + {module, fun, args} -> + Task.async(module, fun, args) + end) + |> Enum.map(fn task -> + Task.await(task, timeout) + end) + end + + @doc """ + Start an external program with apprunner. + + Apprunner handles killing the program if BEAM is abruptly shut down. + Name is used as a prefix for logging output. + + Options that can be given: + + - name: Use as name for logging, otherwise name of binary is used. + - cd: Directory to change to before executing. + + Returns ProgramSpec for the started program. + """ + def exec(executable, args, opts \\ []) do + name = Keyword.get( + opts, + :name, + (executable |> Path.rootname() |> Path.basename()) + ) + + options = [ + :exit_status, # Send msg with status when command stops + args: [Path.join(__DIR__, "apprunner.exs") | [executable | args]], + line: 1024 # Send command output as lines of 1k length + ] + + options = case Keyword.get(opts, :cd) do + nil -> options + cd -> Keyword.put(options, :cd, cd) + end + + Logger.debug("[Spawned] Program #{name}") + + %ProgramSpec{ + name: name, + pending_output: "", + port: Port.open( + {:spawn_executable, @elixir}, + options + ) + } + end + + @doc """ + Start watching a path. Name is used for prefixing logs. + + Path can point to a file or a directory in which case all subdirs will be watched. + + Returns a WatchSpec. + """ + def watch(name, path, fun) do + name_atom = String.to_atom(name) + {:ok, pid} = :fs.start_link(name_atom, String.to_charlist(path)) + :fs.subscribe(name_atom) + + Logger.debug("[Spawned] Watch #{name}") + + %WatchSpec{ + name: name, + path: path, + callback: fun, + pid: pid, + name_atom: name_atom + } + end + + @doc """ + Listen to messages from specs and print them to the screen. + + If watch: true is given in the options, will listen for user's enter key and + kill programs if enter is pressed. + """ + def listen(specs, opts \\ []) + + # If there are no specs, stop running + def listen([], _), do: :ok + + def listen(spec, opts) when not is_list(spec) do + listen([spec], opts) + end + + def listen(specs, opts) when is_list(specs) do + # Start another task to ask for user input if we are in watch mode + task = with \ + true <- Keyword.get(opts, :watch, false), + nil <- Keyword.get(opts, :task), + {:ok, task} <- Task.start_link(__MODULE__, :wait_for_input, [self()]) + do + Logger.info("Programs/watches started, press ENTER to exit.") + task + end + + specs = receive do + # User pressed enter + :user_input_received -> + Logger.info("ENTER received, killing tasks.") + Enum.each(specs, &kill/1) + [] + + # Program sent output with end of line + {port, {:data, {:eol, msg}}} -> + program = Enum.find(specs, program_checker(port)) + msg = :unicode.characters_to_binary(msg, :unicode) + + prefix = "[#{program.name}] #{program.pending_output}" + + Logger.debug(prefix <> msg) + + specs + |> Enum.reject(program_checker(port)) + |> Enum.concat([ + %{program | pending_output: ""} + ]) + + # Program sent output without end of line + {port, {:data, {:noeol, msg}}} -> + program = Enum.find(specs, program_checker(port)) + msg = :unicode.characters_to_binary(msg, :unicode) + + specs + |> Enum.reject(program_checker(port)) + |> Enum.concat([ + %{program | pending_output: "#{program.pending_output}#{msg}"} + ]) + + # Port was closed normally after being told to close + {port, :closed} -> + handle_closed(specs, port) + + # Port closed because the program closed by itself + {port, {:exit_status, 0}} -> + handle_closed(specs, port) + + # Program closed with error status + {port, {:exit_status, status}} -> + program = Enum.find(specs, program_checker(port)) + Logger.error("Program #{program.name} returned status #{status}.") + raise "Failed status #{status} from #{program.name}!" + + # Port crashed + {:EXIT, port, _} -> + handle_closed(specs, port) + + # FS watch sent file event + {_, {:fs, :file_event}, {file, events}} -> + handle_events(specs, file, events) + end + + listen(specs, Keyword.put(opts, :task, task)) + end + + @doc """ + Kill a running program returned by exec(). + """ + def kill(%ProgramSpec{name: name, port: port}) do + if name != nil do + Logger.debug("[Killing] #{name}") + end + + send(port, {self(), :close}) + end + + @doc """ + Kill a running watch. + """ + def kill(%WatchSpec{name: name, pid: pid}) do + Logger.debug("[Killing] #{name}") + Process.exit(pid, :kill) + end + + @doc """ + Print file size of given file in human readable form. + + If old file is given as second argument, print the old file's size + and the diff also. + """ + def print_size(new_file, old_file \\ nil) do + new_size = get_size(new_file) + + {prefix, postfix} = if old_file != nil do + old_size = get_size(old_file) + + { + "#{human_size(old_size)} -> ", + " Diff: #{human_size(old_size - new_size)}" + } + else + {"", ""} + end + + Logger.debug("#{Path.basename(new_file)}: #{prefix}#{human_size(new_size)}.#{postfix}") + end + + def wait_for_input(target) do + IO.gets("") + send(target, :user_input_received) + end + + defp get_size(file) do + %File.Stat{size: size} = File.stat!(file) + size + end + + defp human_size(size) do + size_units = ["B", "kiB", "MiB", "GiB", "TiB", "PiB", "EiB", "ZiB", "YiB"] # You never know + + human_size(size, size_units) + end + + defp human_size(size, [unit | []]), do: size_with_unit(size, unit) + + defp human_size(size, [unit | rest]) do + if size > 1024 do + human_size(size / 1024, rest) + else + size_with_unit(size, unit) + end + end + + defp size_with_unit(size, unit) when is_float(size), do: "#{Float.round(size, 2)} #{unit}" + + defp size_with_unit(size, unit), do: "#{size} #{unit}" + + # Utility to find program in spec list based on port + defp program_checker(port) do + fn + %ProgramSpec{port: program_port} -> + program_port == port + + %WatchSpec{} -> + false + end + end + + # Utility to find watch in spec list based on path + defp watch_checker(path) do + fn + %ProgramSpec{} -> + false + + %WatchSpec{path: watch_path} -> + # If given path is relative to (under) the watch path or is the same + # path completely, it's a match. + path != watch_path and Path.relative_to(path, watch_path) != path + end + end + + defp handle_closed(specs, port) do + case Enum.find(specs, program_checker(port)) do + %ProgramSpec{} = program -> + Logger.debug("[Stopped] #{program.name}") + + specs + |> Enum.reject(program_checker(port)) + + nil -> + # Program was already removed + specs + end + end + + defp handle_events(specs, file, events) do + file = to_string(file) + case Enum.find(specs, watch_checker(file)) do + %WatchSpec{name: name, callback: callback} -> + Logger.debug("[#{name}] Changed #{inspect(events)}: #{file}") + + callback.(file, events) + + nil -> + # Watch was maybe removed for some reason + Logger.debug("[Error] Watch sent event but path was not in specs list: #{inspect(events)} #{file}") + end + + specs + end +end diff --git a/mix.exs b/mix.exs new file mode 100644 index 0000000..db06f91 --- /dev/null +++ b/mix.exs @@ -0,0 +1,36 @@ +defmodule FBU.Mixfile do + use Mix.Project + + def project do + [app: :fbu, + version: "0.1.0", + elixir: "~> 1.4", + build_embedded: Mix.env == :prod, + start_permanent: Mix.env == :prod, + deps: deps()] + end + + # Configuration for the OTP application + # + # Type "mix help compile.app" for more information + def application do + # Specify extra applications you'll use from Erlang/Elixir + [extra_applications: [:logger]] + end + + # Dependencies can be Hex packages: + # + # {:my_dep, "~> 0.3.0"} + # + # Or git/path repositories: + # + # {:my_dep, git: "https://github.com/elixir-lang/my_dep.git", tag: "0.1.0"} + # + # Type "mix help deps" for more examples and options + defp deps do + [ + {:ex_doc, "~> 0.15.0", only: :dev}, + {:fs, "~> 2.12.0"} + ] + end +end diff --git a/mix.lock b/mix.lock new file mode 100644 index 0000000..9316b2e --- /dev/null +++ b/mix.lock @@ -0,0 +1,3 @@ +%{"earmark": {:hex, :earmark, "1.2.0", "bf1ce17aea43ab62f6943b97bd6e3dc032ce45d4f787504e3adf738e54b42f3a", [:mix], []}, + "ex_doc": {:hex, :ex_doc, "0.15.0", "e73333785eef3488cf9144a6e847d3d647e67d02bd6fdac500687854dd5c599f", [:mix], [{:earmark, "~> 1.1", [hex: :earmark, optional: false]}]}, + "fs": {:hex, :fs, "2.12.0", "ad631efacc9a5683c8eaa1b274e24fa64a1b8eb30747e9595b93bec7e492e25e", [:rebar3], []}} diff --git a/test/fbu_test.exs b/test/fbu_test.exs new file mode 100644 index 0000000..ca4d4e4 --- /dev/null +++ b/test/fbu_test.exs @@ -0,0 +1,77 @@ +defmodule FBUTest do + use ExUnit.Case + + # Dependency functions to call in tasks + defmodule Depfuns do + def foo(), do: send(Process.whereis(:deps_test_runner), :foo) + def bar(), do: send(Process.whereis(:deps_test_runner), :bar) + def foo_no(), do: send(Process.whereis(:no_deps_test_runner), :foo) + def bar_no(), do: send(Process.whereis(:no_deps_test_runner), :bar) + end + + test "that a module using the task macro has a run function that takes one argument" do + defmodule Mix.Task.Foo1 do + use FBU.BuildTask + + task args do + assert args == [1] + end + end + + assert Mix.Task.Foo1.run([1]) == :ok + end + + test "that the macro works when given _ to denote ignored args" do + defmodule Mix.Task.Foo2 do + use FBU.BuildTask + + task _ do + :ok + end + end + + assert Mix.Task.Foo2.run([]) == :ok + end + + test "that deps are run before the task" do + defmodule Mix.Task.Foo3 do + use FBU.BuildTask + + @deps [ + &Depfuns.foo/0, + &Depfuns.bar/0 + ] + + task _ do + :ok + end + end + + Process.register(self(), :deps_test_runner) + + Mix.Task.Foo3.run([]) + assert_receive :foo + assert_receive :bar + end + + test "that deps are not run if deps: false is given" do + defmodule Mix.Task.Foo4 do + use FBU.BuildTask + + @deps [ + &Depfuns.foo_no/0, + &Depfuns.bar_no/0 + ] + + task _ do + :ok + end + end + + Process.register(self(), :no_deps_test_runner) + + Mix.Task.Foo4.run([1, 2, 3, 4, 5, deps: false]) + refute_receive :foo + refute_receive :bar + end +end diff --git a/test/test_helper.exs b/test/test_helper.exs new file mode 100644 index 0000000..869559e --- /dev/null +++ b/test/test_helper.exs @@ -0,0 +1 @@ +ExUnit.start()