From 82fd4fc4aba7d3ee3a96bdd394d343105d068416 Mon Sep 17 00:00:00 2001 From: Mikko Ahlroth Date: Wed, 20 Jan 2021 21:57:23 +0200 Subject: [PATCH] Initial commit --- .formatter.exs | 4 + .gitignore | 27 +++++ README.md | 21 ++++ lib/dotenv_parser.ex | 184 ++++++++++++++++++++++++++++++ mix.exs | 30 +++++ mix.lock | 7 ++ test/data/.env | 31 +++++ test/data/.invalid_env | 4 + test/data/.invalid_var_name_chars | 2 + test/data/.invalid_var_name_start | 2 + test/dotenv_parser_test.exs | 61 ++++++++++ test/test_helper.exs | 1 + 12 files changed, 374 insertions(+) create mode 100644 .formatter.exs create mode 100644 .gitignore create mode 100644 README.md create mode 100644 lib/dotenv_parser.ex create mode 100644 mix.exs create mode 100644 mix.lock create mode 100644 test/data/.env create mode 100644 test/data/.invalid_env create mode 100644 test/data/.invalid_var_name_chars create mode 100644 test/data/.invalid_var_name_start create mode 100644 test/dotenv_parser_test.exs create mode 100644 test/test_helper.exs diff --git a/.formatter.exs b/.formatter.exs new file mode 100644 index 0000000..d2cda26 --- /dev/null +++ b/.formatter.exs @@ -0,0 +1,4 @@ +# Used by "mix format" +[ + inputs: ["{mix,.formatter}.exs", "{config,lib,test}/**/*.{ex,exs}"] +] diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..0670801 --- /dev/null +++ b/.gitignore @@ -0,0 +1,27 @@ +# The directory Mix will write compiled artifacts to. +/_build/ + +# If you run "mix test --cover", coverage assets end up here. +/cover/ + +# The directory Mix downloads your dependencies sources to. +/deps/ + +# Where third-party dependencies like ExDoc output generated docs. +/doc/ + +# Ignore .fetch files in case you like to edit your project deps locally. +/.fetch + +# If the VM crashes, it generates a dump, let's ignore it too. +erl_crash.dump + +# Also ignore archive artifacts (built via "mix archive.build"). +*.ez + +# Ignore package tarball (built via "mix hex.build"). +dotenv_parser-*.tar + + +# Temporary files for e.g. tests +/tmp diff --git a/README.md b/README.md new file mode 100644 index 0000000..ea837d4 --- /dev/null +++ b/README.md @@ -0,0 +1,21 @@ +# DotenvParser + +Simple dotenv style file parser that can parse environment data from file and load it. + +See documentation: [https://hexdocs.pm/dotenv_parser](https://hexdocs.pm/dotenv_parser) + +## Installation + +The package can be installed by adding `dotenv_parser` to your list of dependencies in `mix.exs`: + +```elixir +def deps do + [ + {:dotenv_parser, "~> 1.0.0"} + ] +end +``` + +Documentation can be generated with [ExDoc](https://github.com/elixir-lang/ex_doc) +and published on [HexDocs](https://hexdocs.pm). Once published, the docs can +be found at [https://hexdocs.pm/dotenv_parser](https://hexdocs.pm/dotenv_parser). diff --git a/lib/dotenv_parser.ex b/lib/dotenv_parser.ex new file mode 100644 index 0000000..0e38673 --- /dev/null +++ b/lib/dotenv_parser.ex @@ -0,0 +1,184 @@ +defmodule DotenvParser do + @moduledoc """ + Simple parser for dotenv style files. + + Supports simple variable–value pairs with upper and lower cased variables. Values are trimmed of extra whitespace. + + Blank lines and lines starting with `#` are ignored. Additionally inline comments can be added after values with a + `#`, i.e. `FOO=bar # comment`. + + Single quote or double quoted value to prevent trimming of whitespace and allow usage of `#` in value, i.e. `FOO=' bar # not comment ' # comment`. + + Single quoted values don't do any unescaping. Double quoted values will unescape the following: + + * `\\n` - Linefeed + * `\\r` - Carriage return + * `\\t` - Tab + * `\\f` - Form feed + * `\\b` - Backspace + * `\\"` and `\\'` - Quotes + * `\\\\` - Backslash + * `\\uFFFF` - Unicode escape (4 hex characters to denote the codepoint) + + ## Serving suggestion + + If you load lots of environment variables in `config/runtime.exs`, you can easily configure them for development by + having an `.env` file in your development environment and using DotenvParser at the start of the file: + + ```elixir + import Config + + if Config.config_env() == :dev do + DotenvParser.load_file(".env") + end + + # Now variables from `.env` are loaded into system env + config :your_project, + database_url: System.fetch_env!("DB_URL") + """ + + @linefeed_re ~R/\r?\n/ + @line_re ~R/^\s*[a-z_][a-z_0-9]*\s*=/i + @dquoted_val_re ~R/^\s*"(.*)(?!\\)"\s*(?:#.*)?$/ + @squoted_val_re ~R/^\s*'(.*)(?!\\)'\s*(?:#.*)?$/ + @hex_re ~R/^[0-9a-f]+$/i + + @typedoc "Pair of variable name, variable value." + @type value_pair :: {String.t(), String.t()} + + defmodule ParseError do + @moduledoc "Error raised when a line cannot be parsed." + defexception [:message] + end + + @doc """ + Parse given file and load the variables to the environment. + + If a line cannot be parsed or the file cannot be read, an error is raised and no values are loaded to the + environment. + """ + @spec load_file(String.t()) :: :ok + def load_file(file) do + file + |> File.read!() + |> load_data() + end + + @doc """ + Parse given data and load the variables to the environment. + + If a line cannot be parsed, an error is raised and no values are loaded to the environment. + """ + @spec load_data(String.t()) :: :ok + def load_data(data) do + data + |> parse_data() + |> Enum.each(fn {var, val} -> System.put_env(var, val) end) + end + + @doc """ + Parse given file and return a list of variable–value tuples. + + If a line cannot be parsed or the file cannot be read, an error is raised. + """ + @spec parse_file(String.t()) :: [value_pair()] + def parse_file(file) do + file + |> File.read!() + |> parse_data() + end + + @doc """ + Parse given data and return a list of variable–value tuples. + + If a line cannot be parsed, an error is raised. + """ + @spec parse_data(String.t()) :: [value_pair()] + def parse_data(data) do + data + |> String.split(@linefeed_re) + |> Enum.map(&String.trim/1) + |> Enum.reject(&is_comment?/1) + |> Enum.reject(&is_blank?/1) + |> Enum.map(&parse_line/1) + end + + @doc """ + Parse given single line and return a variable–value tuple. + + If line cannot be parsed, an error is raised. + """ + @spec parse_line(String.t()) :: value_pair() + def parse_line(line) do + if not Regex.match?(@line_re, line) do + raise ParseError, "Malformed line cannot be parsed: #{line}" + else + [var, val] = String.split(line, "=", parts: 2) + var = String.trim(var) + val = String.trim(val) + + with {:dquoted, nil} <- {:dquoted, Regex.run(@dquoted_val_re, val)}, + {:squoted, nil} <- {:squoted, Regex.run(@squoted_val_re, val)} do + # Value is plain value + {var, val |> remove_comment() |> String.trim()} + else + {:dquoted, [_, inner_val]} -> {var, stripslashes(inner_val)} + {:squoted, [_, inner_val]} -> {var, inner_val} + end + end + end + + @spec remove_comment(String.t()) :: String.t() + defp remove_comment(val) do + case String.split(val, "#", parts: 2) do + [true_val, _comment] -> true_val + [true_val] -> true_val + end + end + + @spec is_comment?(String.t()) :: boolean() + defp is_comment?(line) + defp is_comment?("#" <> _rest), do: true + defp is_comment?(_line), do: false + + @spec is_blank?(String.t()) :: boolean() + defp is_blank?(line) + defp is_blank?(""), do: true + defp is_blank?(_line), do: false + + @spec stripslashes(String.t(), :slash | :no_slash, String.t()) :: String.t() + defp stripslashes(input, mode \\ :no_slash, acc \\ "") + + defp stripslashes("\\" <> rest, :no_slash, acc) do + stripslashes(rest, :slash, acc) + end + + defp stripslashes("", :no_slash, acc), do: acc + + defp stripslashes(input, :no_slash, acc) do + case String.split(input, "\\", parts: 2) do + [all] -> acc <> all + [head, tail] -> stripslashes(tail, :slash, acc <> head) + end + end + + defp stripslashes("n" <> rest, :slash, acc), do: stripslashes(rest, :no_slash, acc <> "\n") + defp stripslashes("r" <> rest, :slash, acc), do: stripslashes(rest, :no_slash, acc <> "\r") + defp stripslashes("t" <> rest, :slash, acc), do: stripslashes(rest, :no_slash, acc <> "\t") + defp stripslashes("f" <> rest, :slash, acc), do: stripslashes(rest, :no_slash, acc <> "\f") + defp stripslashes("b" <> rest, :slash, acc), do: stripslashes(rest, :no_slash, acc <> "\b") + defp stripslashes("\"" <> rest, :slash, acc), do: stripslashes(rest, :no_slash, acc <> "\"") + defp stripslashes("'" <> rest, :slash, acc), do: stripslashes(rest, :no_slash, acc <> "'") + defp stripslashes("\\" <> rest, :slash, acc), do: stripslashes(rest, :no_slash, acc <> "\\") + + defp stripslashes(<<"u", hex::binary-size(4), rest::binary>>, :slash, acc) do + with true <- Regex.match?(@hex_re, hex), + {int, _rest} <- Integer.parse(hex, 16) do + stripslashes(rest, :no_slash, acc <> <>) + else + _ -> stripslashes(rest, :no_slash, acc <> "\\u" <> hex) + end + end + + defp stripslashes(input, :slash, acc), do: stripslashes(input, :no_slash, acc <> "\\") +end diff --git a/mix.exs b/mix.exs new file mode 100644 index 0000000..ca98e1f --- /dev/null +++ b/mix.exs @@ -0,0 +1,30 @@ +defmodule DotenvParser.MixProject do + use Mix.Project + + def project do + [ + app: :dotenv_parser, + version: "1.0.0", + elixir: "~> 1.11", + start_permanent: Mix.env() == :prod, + deps: deps(), + docs: [ + main: DotenvParser + ] + ] + end + + # Run "mix help compile.app" to learn about applications. + def application do + [ + extra_applications: [:logger] + ] + end + + # Run "mix help deps" to learn about dependencies. + defp deps do + [ + {:ex_doc, "~> 0.23.0", only: :dev} + ] + end +end diff --git a/mix.lock b/mix.lock new file mode 100644 index 0000000..4f6e7c3 --- /dev/null +++ b/mix.lock @@ -0,0 +1,7 @@ +%{ + "earmark_parser": {:hex, :earmark_parser, "1.4.12", "b245e875ec0a311a342320da0551da407d9d2b65d98f7a9597ae078615af3449", [:mix], [], "hexpm", "711e2cc4d64abb7d566d43f54b78f7dc129308a63bc103fbd88550d2174b3160"}, + "ex_doc": {:hex, :ex_doc, "0.23.0", "a069bc9b0bf8efe323ecde8c0d62afc13d308b1fa3d228b65bca5cf8703a529d", [:mix], [{:earmark_parser, "~> 1.4.0", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.14", [hex: :makeup_elixir, repo: "hexpm", optional: false]}], "hexpm", "f5e2c4702468b2fd11b10d39416ddadd2fcdd173ba2a0285ebd92c39827a5a16"}, + "makeup": {:hex, :makeup, "1.0.5", "d5a830bc42c9800ce07dd97fa94669dfb93d3bf5fcf6ea7a0c67b2e0e4a7f26c", [:mix], [{:nimble_parsec, "~> 0.5 or ~> 1.0", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "cfa158c02d3f5c0c665d0af11512fed3fba0144cf1aadee0f2ce17747fba2ca9"}, + "makeup_elixir": {:hex, :makeup_elixir, "0.15.0", "98312c9f0d3730fde4049985a1105da5155bfe5c11e47bdc7406d88e01e4219b", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.1", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "75ffa34ab1056b7e24844c90bfc62aaf6f3a37a15faa76b07bc5eba27e4a8b4a"}, + "nimble_parsec": {:hex, :nimble_parsec, "1.1.0", "3a6fca1550363552e54c216debb6a9e95bd8d32348938e13de5eda962c0d7f89", [:mix], [], "hexpm", "08eb32d66b706e913ff748f11694b17981c0b04a33ef470e33e11b3d3ac8f54b"}, +} diff --git a/test/data/.env b/test/data/.env new file mode 100644 index 0000000..50c7676 --- /dev/null +++ b/test/data/.env @@ -0,0 +1,31 @@ +BASIC=basic + +# previous line intentionally left blank + # Indented comment +AFTER_LINE=after_line +EMPTY= +SINGLE_QUOTES='single_quotes' +SINGLE_QUOTES_SPACED=' single quotes ' +DOUBLE_QUOTES="double_quotes" +DOUBLE_QUOTES_SPACED=" double quotes " +EXPAND_NEWLINES="expand\nnew\nlines" +EXPAND_MANY_SLASHES="expand\\\\nmany \tslashes" +EXPAND_UNICODE="expand \u1234" +INVALID_UNICODE="not valid \uarf1" +DONT_EXPAND_UNQUOTED=dontexpand\nnewlines\u1234 +DONT_EXPAND_SQUOTED='dontexpand\nnewlines' +# COMMENTS=work +EQUAL_SIGNS=equals== +RETAIN_INNER_QUOTES={"foo": "bar"} +RETAIN_LEADING_DQUOTE="retained +RETAIN_LEADING_SQUOTE='retained +RETAIN_TRAILING_DQUOTE=retained" +RETAIN_TRAILING_SQUOTE=retained' +RETAIN_INNER_QUOTES_AS_STRING='{"foo": "bar"}' +TRIM_SPACE_FROM_UNQUOTED= some spaced out string +USERNAME=therealnerdybeast@example.tld + SPACED_KEY = parsed +INLINE_COMMENT="foo#bar" # Bark Bark +INLINE_COMMENT_PLAIN=foo bar # Bark Bark +END_BACKSLASH="something\" # Comment +lowercased_var=foo diff --git a/test/data/.invalid_env b/test/data/.invalid_env new file mode 100644 index 0000000..6f64b9d --- /dev/null +++ b/test/data/.invalid_env @@ -0,0 +1,4 @@ +FOO=bar + +# This line will not parse +ARGH diff --git a/test/data/.invalid_var_name_chars b/test/data/.invalid_var_name_chars new file mode 100644 index 0000000..6480ed2 --- /dev/null +++ b/test/data/.invalid_var_name_chars @@ -0,0 +1,2 @@ +# Var name contains invalid chars +FOO-BAR=baz diff --git a/test/data/.invalid_var_name_start b/test/data/.invalid_var_name_start new file mode 100644 index 0000000..d42688d --- /dev/null +++ b/test/data/.invalid_var_name_start @@ -0,0 +1,2 @@ +# Var name starting with number is not allowed +0FOO=bar diff --git a/test/dotenv_parser_test.exs b/test/dotenv_parser_test.exs new file mode 100644 index 0000000..c5ad954 --- /dev/null +++ b/test/dotenv_parser_test.exs @@ -0,0 +1,61 @@ +defmodule DotenvParserTest do + use ExUnit.Case + doctest DotenvParser + + test "parses correctly" do + assert DotenvParser.parse_file("test/data/.env") == [ + {"BASIC", "basic"}, + {"AFTER_LINE", "after_line"}, + {"EMPTY", ""}, + {"SINGLE_QUOTES", "single_quotes"}, + {"SINGLE_QUOTES_SPACED", " single quotes "}, + {"DOUBLE_QUOTES", "double_quotes"}, + {"DOUBLE_QUOTES_SPACED", " double quotes "}, + {"EXPAND_NEWLINES", "expand\nnew\nlines"}, + {"EXPAND_MANY_SLASHES", "expand\\\\nmany \tslashes"}, + {"EXPAND_UNICODE", "expand ሴ"}, + {"INVALID_UNICODE", "not valid \\uarf1"}, + {"DONT_EXPAND_UNQUOTED", "dontexpand\\nnewlines\\u1234"}, + {"DONT_EXPAND_SQUOTED", "dontexpand\\nnewlines"}, + {"EQUAL_SIGNS", "equals=="}, + {"RETAIN_INNER_QUOTES", "{\"foo\": \"bar\"}"}, + {"RETAIN_LEADING_DQUOTE", "\"retained"}, + {"RETAIN_LEADING_SQUOTE", "'retained"}, + {"RETAIN_TRAILING_DQUOTE", "retained\""}, + {"RETAIN_TRAILING_SQUOTE", "retained'"}, + {"RETAIN_INNER_QUOTES_AS_STRING", "{\"foo\": \"bar\"}"}, + {"TRIM_SPACE_FROM_UNQUOTED", "some spaced out string"}, + {"USERNAME", "therealnerdybeast@example.tld"}, + {"SPACED_KEY", "parsed"}, + {"INLINE_COMMENT", "foo#bar"}, + {"INLINE_COMMENT_PLAIN", "foo bar"}, + {"END_BACKSLASH", "something\\"}, + {"lowercased_var", "foo"} + ] + end + + test "loads env" do + DotenvParser.load_file("test/data/.env") + + assert System.get_env("INLINE_COMMENT") == "foo#bar" + assert System.get_env("lowercased_var") == "foo" + end + + test "fails with invalid env and no env is loaded" do + assert_raise(DotenvParser.ParseError, fn -> + DotenvParser.load_file("test/data/.invalid_env") + end) + + assert is_nil(System.get_env("FOO")) + end + + test "fails with invalid variable names" do + assert_raise(DotenvParser.ParseError, fn -> + DotenvParser.parse_file("test/data/.invalid_var_name_start") + end) + + assert_raise(DotenvParser.ParseError, fn -> + DotenvParser.parse_file("test/data/.invalid_var_name_chars") + end) + end +end diff --git a/test/test_helper.exs b/test/test_helper.exs new file mode 100644 index 0000000..869559e --- /dev/null +++ b/test/test_helper.exs @@ -0,0 +1 @@ +ExUnit.start()