Merge branch 'feature/multiline-values' into 'trunk'
Feature: multiline values Closes #1 See merge request Nicd/dotenv-parser!1
This commit is contained in:
commit
735e1867f6
9 changed files with 216 additions and 36 deletions
2
.tool-versions
Normal file
2
.tool-versions
Normal file
|
@ -0,0 +1,2 @@
|
|||
elixir 1.11.3-otp-23
|
||||
erlang 23.2.2
|
10
CHANGELOG
Normal file
10
CHANGELOG
Normal file
|
@ -0,0 +1,10 @@
|
|||
2.0.0
|
||||
-----
|
||||
|
||||
+ Added support for multiline values by using quotes around the value.
|
||||
* Fixed some escaping related corner cases.
|
||||
|
||||
1.2.0
|
||||
-----
|
||||
|
||||
First public release.
|
|
@ -11,7 +11,7 @@ The package can be installed by adding `dotenv_parser` to your list of dependenc
|
|||
```elixir
|
||||
def deps do
|
||||
[
|
||||
{:dotenv_parser, "~> 1.2"}
|
||||
{:dotenv_parser, "~> 2.0"}
|
||||
]
|
||||
end
|
||||
```
|
||||
|
|
|
@ -7,7 +7,7 @@ defmodule DotenvParser do
|
|||
Blank lines and lines starting with `#` are ignored. Additionally inline comments can be added after values with a
|
||||
`#`, i.e. `FOO=bar # comment`.
|
||||
|
||||
Single quote or double quoted value to prevent trimming of whitespace and allow usage of `#` in value, i.e. `FOO=' bar # not comment ' # comment`.
|
||||
Single quote or double quote value to prevent trimming of whitespace and allow usage of `#` in value, i.e. `FOO=' bar # not comment ' # comment`.
|
||||
|
||||
Single quoted values don't do any unescaping. Double quoted values will unescape the following:
|
||||
|
||||
|
@ -19,6 +19,20 @@ defmodule DotenvParser do
|
|||
* `\\"` and `\\'` - Quotes
|
||||
* `\\\\` - Backslash
|
||||
* `\\uFFFF` - Unicode escape (4 hex characters to denote the codepoint)
|
||||
* A backslash at the end of the line in a multiline value will remove the linefeed.
|
||||
|
||||
Values can span multiple lines when single or double quoted:
|
||||
|
||||
```sh
|
||||
MULTILINE="This is a
|
||||
multiline value."
|
||||
```
|
||||
|
||||
This will result in the following:
|
||||
|
||||
```elixir
|
||||
System.fetch_env!("MULTILINE") == "This is a\\nmultiline value."
|
||||
```
|
||||
|
||||
A line can start with `export ` for easier interoperation with regular shell scripts. These lines are treated the
|
||||
same as any others.
|
||||
|
@ -42,10 +56,14 @@ defmodule DotenvParser do
|
|||
|
||||
@linefeed_re ~R/\r?\n/
|
||||
@line_re ~R/^(?:\s*export)?\s*[a-z_][a-z_0-9]*\s*=/i
|
||||
@dquoted_val_re ~R/^\s*"(.*)(?!\\)"\s*(?:#.*)?$/
|
||||
@squoted_val_re ~R/^\s*'(.*)(?!\\)'\s*(?:#.*)?$/
|
||||
@dquoted_val_re ~R/^"([^"\\]*(?:\\.[^"\\]*)*)"\s*(?:#.*)?$/
|
||||
@squoted_val_re ~R/^\s*'(.*)'\s*(?:#.*)?$/
|
||||
@dquoted_multiline_end ~R/^([^"\\]*(?:\\.[^"\\]*)*)"\s*(?:#.*)?$/
|
||||
@squoted_multiline_end ~R/^(.*)'\s*(?:#.*)?$/
|
||||
@hex_re ~R/^[0-9a-f]+$/i
|
||||
|
||||
@quote_chars ~w(" ')
|
||||
|
||||
@typedoc "Pair of variable name, variable value."
|
||||
@type value_pair :: {String.t(), String.t()}
|
||||
|
||||
|
@ -54,6 +72,20 @@ defmodule DotenvParser do
|
|||
defexception [:message]
|
||||
end
|
||||
|
||||
defmodule Continuation do
|
||||
@typedoc """
|
||||
A multiline value continuation. When a function returns this, it means that a multiline value
|
||||
was started and more needs to be parsed to get the rest of the value.
|
||||
"""
|
||||
@type t :: %__MODULE__{
|
||||
name: String.t(),
|
||||
value: String.t(),
|
||||
start_quote: String.t()
|
||||
}
|
||||
@enforce_keys [:name, :value, :start_quote]
|
||||
defstruct [:name, :value, :start_quote]
|
||||
end
|
||||
|
||||
@doc """
|
||||
Parse given file and load the variables to the environment.
|
||||
|
||||
|
@ -98,39 +130,124 @@ defmodule DotenvParser do
|
|||
"""
|
||||
@spec parse_data(String.t()) :: [value_pair()]
|
||||
def parse_data(data) do
|
||||
{value_pairs, continuation} =
|
||||
data
|
||||
|> String.split(@linefeed_re)
|
||||
|> Enum.map(&String.trim/1)
|
||||
|> Enum.reject(&is_comment?/1)
|
||||
|> Enum.reject(&is_blank?/1)
|
||||
|> Enum.map(&parse_line/1)
|
||||
|> Enum.reduce({[], nil}, fn
|
||||
line, {ret, nil} ->
|
||||
trimmed = String.trim(line)
|
||||
|
||||
if not is_comment?(trimmed) and not is_blank?(trimmed) do
|
||||
reduce_line(ret, line, nil)
|
||||
else
|
||||
{ret, nil}
|
||||
end
|
||||
|
||||
line, {ret, continuation} ->
|
||||
reduce_line(ret, line, continuation)
|
||||
end)
|
||||
|
||||
if not is_nil(continuation) do
|
||||
raise ParseError,
|
||||
"Could not find end for quote #{continuation.start_quote} in variable #{continuation.name}"
|
||||
end
|
||||
|
||||
Enum.reverse(value_pairs)
|
||||
end
|
||||
|
||||
@doc """
|
||||
Parse given single line and return a variable–value tuple.
|
||||
Parse given single line and return a variable–value tuple, or a continuation value if the line
|
||||
started or continued a multiline value.
|
||||
|
||||
If line cannot be parsed, an error is raised.
|
||||
|
||||
The second argument needs to be `nil` or a continuation value returned from parsing the previous
|
||||
line.
|
||||
"""
|
||||
@spec parse_line(String.t()) :: value_pair()
|
||||
def parse_line(line) do
|
||||
@spec parse_line(String.t(), Continuation.t() | nil) :: value_pair() | Continuation.t()
|
||||
def parse_line(line, state)
|
||||
|
||||
def parse_line(line, nil) do
|
||||
if not Regex.match?(@line_re, line) do
|
||||
raise ParseError, "Malformed line cannot be parsed: #{line}"
|
||||
else
|
||||
[var, val] = String.split(line, "=", parts: 2)
|
||||
var = var |> String.trim() |> String.replace_leading("export ", "")
|
||||
val = String.trim(val)
|
||||
trimmed = String.trim(val)
|
||||
|
||||
with {:dquoted, nil} <- {:dquoted, Regex.run(@dquoted_val_re, val)},
|
||||
{:squoted, nil} <- {:squoted, Regex.run(@squoted_val_re, val)} do
|
||||
with {:dquoted, nil} <- {:dquoted, Regex.run(@dquoted_val_re, trimmed)},
|
||||
{:squoted, nil} <- {:squoted, Regex.run(@squoted_val_re, trimmed)},
|
||||
trimmed_leading = String.trim_leading(val),
|
||||
{:quoted_start, false} <-
|
||||
{:quoted_start, String.starts_with?(trimmed_leading, @quote_chars)} do
|
||||
# Value is plain value
|
||||
{var, val |> remove_comment() |> String.trim()}
|
||||
{var, trimmed |> remove_comment() |> String.trim()}
|
||||
else
|
||||
{:dquoted, [_, inner_val]} -> {var, stripslashes(inner_val)}
|
||||
{:squoted, [_, inner_val]} -> {var, inner_val}
|
||||
{:dquoted, [_, inner_val]} ->
|
||||
{var, stripslashes(inner_val)}
|
||||
|
||||
{:squoted, [_, inner_val]} ->
|
||||
{var, inner_val}
|
||||
|
||||
{:quoted_start, _} ->
|
||||
parse_multiline_start(var, val)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def parse_line(line, %Continuation{} = continuation) do
|
||||
trimmed = String.trim_trailing(line)
|
||||
|
||||
end_match =
|
||||
if continuation.start_quote == "\"" do
|
||||
Regex.run(@dquoted_multiline_end, trimmed)
|
||||
else
|
||||
Regex.run(@squoted_multiline_end, trimmed)
|
||||
end
|
||||
|
||||
with [_, line_content] <- end_match do
|
||||
ret = maybe_stripslashes(continuation, line_content)
|
||||
{continuation.name, continuation.value <> ret}
|
||||
else
|
||||
_ ->
|
||||
next_line = maybe_stripslashes(continuation, line)
|
||||
next_line = maybe_linefeed(continuation, next_line)
|
||||
|
||||
%Continuation{
|
||||
continuation
|
||||
| value: continuation.value <> next_line
|
||||
}
|
||||
end
|
||||
end
|
||||
|
||||
@spec parse_multiline_start(String.t(), String.t()) :: Continuation.t()
|
||||
defp parse_multiline_start(name, input) do
|
||||
{start_quote, rest} = input |> String.trim_leading() |> String.split_at(1)
|
||||
|
||||
continuation = %Continuation{
|
||||
name: name,
|
||||
value: "",
|
||||
start_quote: start_quote
|
||||
}
|
||||
|
||||
value = maybe_stripslashes(continuation, rest)
|
||||
value = maybe_linefeed(continuation, value)
|
||||
|
||||
%Continuation{continuation | value: value}
|
||||
end
|
||||
|
||||
@spec reduce_line([value_pair()], String.t(), Continuation.t() | nil) ::
|
||||
{[value_pair()], Continuation.t() | nil}
|
||||
defp reduce_line(ret, line, continuation) do
|
||||
case parse_line(line, continuation) do
|
||||
%Continuation{} = new_continuation ->
|
||||
{ret, new_continuation}
|
||||
|
||||
result ->
|
||||
{[result | ret], nil}
|
||||
end
|
||||
end
|
||||
|
||||
@spec remove_comment(String.t()) :: String.t()
|
||||
defp remove_comment(val) do
|
||||
case String.split(val, "#", parts: 2) do
|
||||
|
@ -184,4 +301,23 @@ defmodule DotenvParser do
|
|||
end
|
||||
|
||||
defp stripslashes(input, :slash, acc), do: stripslashes(input, :no_slash, acc <> "\\")
|
||||
|
||||
@spec maybe_stripslashes(Continuation.t(), String.t()) :: String.t()
|
||||
defp maybe_stripslashes(continuation, input)
|
||||
|
||||
defp maybe_stripslashes(%Continuation{start_quote: "\""}, input), do: stripslashes(input)
|
||||
defp maybe_stripslashes(_, input), do: input
|
||||
|
||||
@spec maybe_linefeed(Continuation.t(), String.t()) :: String.t()
|
||||
defp maybe_linefeed(continuation, input)
|
||||
|
||||
defp maybe_linefeed(%Continuation{start_quote: "\""}, input) do
|
||||
if String.ends_with?(input, "\\") do
|
||||
String.slice(input, 0..-2)
|
||||
else
|
||||
input <> "\n"
|
||||
end
|
||||
end
|
||||
|
||||
defp maybe_linefeed(_, input), do: input <> "\n"
|
||||
end
|
||||
|
|
4
mix.exs
4
mix.exs
|
@ -4,7 +4,7 @@ defmodule DotenvParser.MixProject do
|
|||
def project do
|
||||
[
|
||||
app: :dotenv_parser,
|
||||
version: "1.2.0",
|
||||
version: "2.0.0",
|
||||
elixir: "~> 1.11",
|
||||
start_permanent: Mix.env() == :prod,
|
||||
deps: deps(),
|
||||
|
@ -31,7 +31,7 @@ defmodule DotenvParser.MixProject do
|
|||
# Run "mix help deps" to learn about dependencies.
|
||||
defp deps do
|
||||
[
|
||||
{:ex_doc, "~> 0.23.0", only: :dev}
|
||||
{:ex_doc, "~> 0.28.0", only: :dev}
|
||||
]
|
||||
end
|
||||
end
|
||||
|
|
9
mix.lock
9
mix.lock
|
@ -1,7 +1,8 @@
|
|||
%{
|
||||
"earmark_parser": {:hex, :earmark_parser, "1.4.12", "b245e875ec0a311a342320da0551da407d9d2b65d98f7a9597ae078615af3449", [:mix], [], "hexpm", "711e2cc4d64abb7d566d43f54b78f7dc129308a63bc103fbd88550d2174b3160"},
|
||||
"ex_doc": {:hex, :ex_doc, "0.23.0", "a069bc9b0bf8efe323ecde8c0d62afc13d308b1fa3d228b65bca5cf8703a529d", [:mix], [{:earmark_parser, "~> 1.4.0", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.14", [hex: :makeup_elixir, repo: "hexpm", optional: false]}], "hexpm", "f5e2c4702468b2fd11b10d39416ddadd2fcdd173ba2a0285ebd92c39827a5a16"},
|
||||
"earmark_parser": {:hex, :earmark_parser, "1.4.19", "de0d033d5ff9fc396a24eadc2fcf2afa3d120841eb3f1004d138cbf9273210e8", [:mix], [], "hexpm", "527ab6630b5c75c3a3960b75844c314ec305c76d9899bb30f71cb85952a9dc45"},
|
||||
"ex_doc": {:hex, :ex_doc, "0.28.0", "7eaf526dd8c80ae8c04d52ac8801594426ae322b52a6156cd038f30bafa8226f", [:mix], [{:earmark_parser, "~> 1.4.19", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.14", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1", [hex: :makeup_erlang, repo: "hexpm", optional: false]}], "hexpm", "e55cdadf69a5d1f4cfd8477122ebac5e1fadd433a8c1022dafc5025e48db0131"},
|
||||
"makeup": {:hex, :makeup, "1.0.5", "d5a830bc42c9800ce07dd97fa94669dfb93d3bf5fcf6ea7a0c67b2e0e4a7f26c", [:mix], [{:nimble_parsec, "~> 0.5 or ~> 1.0", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "cfa158c02d3f5c0c665d0af11512fed3fba0144cf1aadee0f2ce17747fba2ca9"},
|
||||
"makeup_elixir": {:hex, :makeup_elixir, "0.15.0", "98312c9f0d3730fde4049985a1105da5155bfe5c11e47bdc7406d88e01e4219b", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.1", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "75ffa34ab1056b7e24844c90bfc62aaf6f3a37a15faa76b07bc5eba27e4a8b4a"},
|
||||
"nimble_parsec": {:hex, :nimble_parsec, "1.1.0", "3a6fca1550363552e54c216debb6a9e95bd8d32348938e13de5eda962c0d7f89", [:mix], [], "hexpm", "08eb32d66b706e913ff748f11694b17981c0b04a33ef470e33e11b3d3ac8f54b"},
|
||||
"makeup_elixir": {:hex, :makeup_elixir, "0.15.2", "dc72dfe17eb240552857465cc00cce390960d9a0c055c4ccd38b70629227e97c", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.1", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "fd23ae48d09b32eff49d4ced2b43c9f086d402ee4fd4fcb2d7fad97fa8823e75"},
|
||||
"makeup_erlang": {:hex, :makeup_erlang, "0.1.1", "3fcb7f09eb9d98dc4d208f49cc955a34218fc41ff6b84df7c75b3e6e533cc65f", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "174d0809e98a4ef0b3309256cbf97101c6ec01c4ab0b23e926a9e17df2077cbb"},
|
||||
"nimble_parsec": {:hex, :nimble_parsec, "1.2.1", "264fc6864936b59fedb3ceb89998c64e9bb91945faf1eb115d349b96913cc2ef", [:mix], [], "hexpm", "23c31d0ec38c97bf9adde35bc91bc8e1181ea5202881f48a192f4aa2d2cf4d59"},
|
||||
}
|
||||
|
|
|
@ -17,16 +17,30 @@ DONT_EXPAND_SQUOTED='dontexpand\nnewlines'
|
|||
# COMMENTS=work
|
||||
EQUAL_SIGNS=equals==
|
||||
RETAIN_INNER_QUOTES={"foo": "bar"}
|
||||
RETAIN_LEADING_DQUOTE="retained
|
||||
RETAIN_LEADING_SQUOTE='retained
|
||||
RETAIN_TRAILING_DQUOTE=retained"
|
||||
RETAIN_TRAILING_SQUOTE=retained'
|
||||
RETAIN_INNER_QUOTES_AS_STRING='{"foo": "bar"}'
|
||||
TRIM_SPACE_FROM_UNQUOTED= some spaced out string
|
||||
USERNAME=therealnerdybeast@example.tld
|
||||
SPACED_KEY = parsed
|
||||
INLINE_COMMENT="foo#bar" # Bark Bark
|
||||
INLINE_COMMENT_PLAIN=foo bar # Bark Bark
|
||||
END_BACKSLASH="something\" # Comment
|
||||
END_BACKSLASH="something\" # Comment"
|
||||
END_DOUBLE_BACKSLASH="foo\\" # Comment
|
||||
lowercased_var=foo
|
||||
export FOO="exports are supported"
|
||||
|
||||
MULTILINE="There are
|
||||
many\u1234lines
|
||||
in this var!"
|
||||
SPACED_MULTILINE = "
|
||||
<-- Note: Indentation preserved"
|
||||
MULTILINE_SINGLE_QUOTE='Now also
|
||||
with single quotes! "_"'
|
||||
export MULTILINE_ESCAPED_QUOTE="Not stopping here --> \\\\\\\"
|
||||
but \"\" here" # Comment for multiline
|
||||
MULTILINE_ESCAPED_LINEFEED="No linefeed --> \
|
||||
nope"
|
||||
export MULTILINE_NON_ESCAPED_LINEFEED='Linefeed not escaped in single quotes \
|
||||
'
|
||||
|
||||
RETAIN_TRAILING_DQUOTE=retained"
|
||||
RETAIN_TRAILING_SQUOTE=retained'
|
||||
|
|
6
test/data/.invalid_multiline_end
Normal file
6
test/data/.invalid_multiline_end
Normal file
|
@ -0,0 +1,6 @@
|
|||
# No end found for multiline string
|
||||
|
||||
ENV=Cool
|
||||
BEANS="I like beans
|
||||
MULTILINE_ESCAPED=\"
|
||||
PEAS=0
|
|
@ -53,19 +53,24 @@ defmodule DotenvParserTest do
|
|||
{"DONT_EXPAND_SQUOTED", "dontexpand\\nnewlines"},
|
||||
{"EQUAL_SIGNS", "equals=="},
|
||||
{"RETAIN_INNER_QUOTES", "{\"foo\": \"bar\"}"},
|
||||
{"RETAIN_LEADING_DQUOTE", "\"retained"},
|
||||
{"RETAIN_LEADING_SQUOTE", "'retained"},
|
||||
{"RETAIN_TRAILING_DQUOTE", "retained\""},
|
||||
{"RETAIN_TRAILING_SQUOTE", "retained'"},
|
||||
{"RETAIN_INNER_QUOTES_AS_STRING", "{\"foo\": \"bar\"}"},
|
||||
{"TRIM_SPACE_FROM_UNQUOTED", "some spaced out string"},
|
||||
{"USERNAME", "therealnerdybeast@example.tld"},
|
||||
{"SPACED_KEY", "parsed"},
|
||||
{"INLINE_COMMENT", "foo#bar"},
|
||||
{"INLINE_COMMENT_PLAIN", "foo bar"},
|
||||
{"END_BACKSLASH", "something\\"},
|
||||
{"END_BACKSLASH", "something\" # Comment"},
|
||||
{"END_DOUBLE_BACKSLASH", "foo\\"},
|
||||
{"lowercased_var", "foo"},
|
||||
{"FOO", "exports are supported"}
|
||||
{"FOO", "exports are supported"},
|
||||
{"MULTILINE", "There are\nmanyሴlines\nin this var!"},
|
||||
{"SPACED_MULTILINE", " \n <-- Note: Indentation preserved"},
|
||||
{"MULTILINE_SINGLE_QUOTE", "Now also\nwith single quotes! \"_\""},
|
||||
{"MULTILINE_ESCAPED_QUOTE", "Not stopping here --> \\\\\\\"\nbut \"\" here"},
|
||||
{"MULTILINE_ESCAPED_LINEFEED", "No linefeed --> nope"},
|
||||
{"MULTILINE_NON_ESCAPED_LINEFEED", "Linefeed not escaped in single quotes \\\n"},
|
||||
{"RETAIN_TRAILING_DQUOTE", "retained\""},
|
||||
{"RETAIN_TRAILING_SQUOTE", "retained'"}
|
||||
]
|
||||
end
|
||||
|
||||
|
@ -93,4 +98,10 @@ defmodule DotenvParserTest do
|
|||
DotenvParser.parse_file("test/data/.invalid_var_name_chars")
|
||||
end)
|
||||
end
|
||||
|
||||
test "fails with invalid multiline variable" do
|
||||
assert_raise(DotenvParser.ParseError, fn ->
|
||||
DotenvParser.parse_file("test/data/.invalid_multiline_end")
|
||||
end)
|
||||
end
|
||||
end
|
||||
|
|
Loading…
Reference in a new issue