package confi
import (
"encoding/gob"
"encoding/json"
"io"
"github.com/pkg/errors"
"gopkg.in/yaml.v3"
)
type Codec struct {
MarshalFn func(value any, writer io.Writer) error
UnmarshalFn func(reader io.Reader, value any) error
}
func (c Codec) Marshal(value any, writer io.Writer) error {
data, err := yaml.Marshal(value)
if err != nil {
return errors.Wrap(err, "marshal value to yaml")
}
values := make(map[string]any)
if err := yaml.Unmarshal(data, &values); err != nil {
return errors.Wrap(err, "unmarshal values from yaml")
}
if err := c.MarshalFn(values, writer); err != nil {
return errors.Wrap(err, "marshal values")
}
return nil
}
func (c Codec) Unmarshal(reader io.Reader, value any) error {
values := make(map[string]any)
if err := c.UnmarshalFn(reader, &values); err != nil {
return errors.Wrap(err, "unmarshal values")
}
data, err := yaml.Marshal(values)
if err != nil {
return errors.Wrap(err, "marshal values to yaml")
}
if err := yaml.Unmarshal(data, value); err != nil {
return errors.Wrap(err, "unmarshal value from yaml")
}
return nil
}
var JSON = Codec{
MarshalFn: func(value any, writer io.Writer) error {
encoder := json.NewEncoder(writer)
encoder.SetIndent("", " ")
return encoder.Encode(value)
},
UnmarshalFn: func(reader io.Reader, value any) error { return json.NewDecoder(reader).Decode(value) },
}
var YAML = Codec{
MarshalFn: func(value any, writer io.Writer) error {
encoder := yaml.NewEncoder(writer)
encoder.SetIndent(2)
return encoder.Encode(value)
},
UnmarshalFn: func(reader io.Reader, value any) error { return yaml.NewDecoder(reader).Decode(value) },
}
var Gob = Codec{
MarshalFn: func(value any, writer io.Writer) error { return gob.NewEncoder(writer).Encode(value) },
UnmarshalFn: func(reader io.Reader, value any) error { return gob.NewDecoder(reader).Decode(value) },
}
var Codecs = map[string]Codec{
"json": JSON,
"yaml": YAML,
"yml": YAML,
"gob": Gob,
}
type Format string
func (f Format) SchemaEnum() any {
var formats []string
for format := range Codecs {
formats = append(formats, format)
}
return formats
}
package confi
import (
"context"
"os"
"strconv"
"strings"
"github.com/pkg/errors"
"gopkg.in/yaml.v3"
)
func Get[T any](ctx context.Context, appName string) (*T, *Schema, error) {
replacer := strings.NewReplacer(`-`, `_`, `.`, `_`)
provider := &DefaultSourceProvider{
EnvPrefix: replacer.Replace(appName) + "_",
Env: os.Environ(),
Args: os.Args[1:],
Stdin: os.Stdin,
}
return FromProvider[T](ctx, provider)
}
func FromProvider[T any](ctx context.Context, provider SourceProvider) (*T, *Schema, error) {
sources, err := provider.GetSources(ctx)
if err != nil {
return nil, nil, errors.Wrap(err, "get sources")
}
var config T
schema, err := GenerateSchema(config)
if err != nil {
return nil, nil, errors.Wrapf(err, "generate schema")
}
for _, source := range sources {
values, err := source.GetValues(ctx)
if err != nil {
return nil, nil, errors.Wrapf(err, "get values from %s", source)
}
data, err := yaml.Marshal(SpecifyType(values))
if err != nil {
return nil, nil, errors.Wrapf(err, "marshal values from %s to yaml", source)
}
if err := yaml.Unmarshal(data, &config); err != nil {
return nil, nil, errors.Wrapf(err, "unmarshal values from %s from yaml", source)
}
}
if err := schema.ApplyDefaults(&config); err != nil {
return nil, nil, errors.Wrap(err, "apply defaults")
}
return &config, schema, nil
}
func testFloat(source string, target float64) bool {
tokens := strings.Split(source, ".")
var prec int
if len(tokens) > 1 {
prec = len(tokens[1])
}
return strconv.FormatFloat(target, 'f', prec, 64) == source
}
func SpecifyType(source any) any {
switch source := source.(type) {
case string:
if target, err := strconv.ParseBool(source); err == nil && strconv.FormatBool(target) == source {
return true
} else if target, err := strconv.ParseUint(source, 10, 64); err == nil && strconv.FormatUint(target, 10) == source {
return target
} else if target, err := strconv.ParseInt(source, 10, 64); err == nil && strconv.FormatInt(target, 10) == source {
return target
} else if target, err := strconv.ParseFloat(source, 64); err == nil && testFloat(source, target) {
return target
}
return source
case []any:
target := make([]any, len(source))
for i, value := range source {
target[i] = SpecifyType(value)
}
return target
case map[string]any:
target := make(map[any]any, len(source))
for key, value := range source {
target[SpecifyType(key)] = SpecifyType(value)
}
return target
}
return source
}
package confi
import (
"io"
"os"
)
type Input interface {
Reader() (io.Reader, error)
}
type File string
func (f File) Path() string { return string(f) }
func (f File) Reader() (io.Reader, error) { return os.Open(f.Path()) }
type Reader struct {
R io.Reader
}
func (r Reader) Reader() (io.Reader, error) { return r.R, nil }
func CloseQuietly(value any) {
if value == os.Stdin || value == os.Stdout || value == os.Stderr {
return
}
if closer, ok := value.(io.Closer); ok {
_ = closer.Close()
}
}
package confi
import (
"bufio"
"io"
"strings"
"github.com/pkg/errors"
)
type Property struct {
Path []string
Value string
}
func (p Property) Key() string {
return strings.Join(p.Path, ".")
}
func getProperty(text, sep string, expandBools bool) (*Property, error) {
tokens := strings.SplitN(text, "=", 2)
if len(tokens[0]) == 0 {
return nil, errors.New("empty property name")
}
if len(tokens) == 1 {
if expandBools {
tokens = append(tokens, "true")
} else {
return nil, errors.Errorf("empty property value")
}
}
return &Property{
Path: strings.Split(tokens[0], sep),
Value: tokens[1],
}, nil
}
func readProperties(reader io.Reader) ([]Property, error) {
var props []Property
scanner := bufio.NewScanner(reader)
for scanner.Scan() {
line := strings.Trim(scanner.Text(), " ")
if line == "" || strings.HasPrefix(line, "#") {
continue
}
prop, err := getProperty(line, ".", false)
if err != nil {
return nil, errors.Wrapf(err, `"%s""`, line)
}
props = append(props, *prop)
}
return props, nil
}
package confi
import (
"reflect"
"strings"
"time"
"github.com/AlekSi/pointer"
"github.com/pkg/errors"
"gopkg.in/yaml.v3"
)
type formatted interface {
SchemaFormat() string
}
type patterned interface {
SchemaPattern() string
}
type enum interface {
SchemaEnum() any
}
type Schema struct {
Type string `yaml:"type"`
Items *Schema `yaml:"items,omitempty"`
Properties map[string]Schema `yaml:"properties,omitempty"`
AdditionalProperties any `yaml:"additionalProperties,omitempty"`
Required []string `yaml:"required,omitempty"`
// properties below are applied to primitive or inner types
Enum any `yaml:"enum,omitempty" prop:"inner,array"`
Examples any `yaml:"examples,omitempty" prop:"inner,array"`
Pattern string `yaml:"pattern,omitempty" prop:"inner"`
Format string `yaml:"format,omitempty" prop:"inner" alias:"fmt"`
Minimum any `yaml:"minimum,omitempty" prop:"inner" alias:"min"`
ExclusiveMinimum any `yaml:"exclusiveMinimum,omitempty" prop:"inner" alias:"xmin"`
Maximum any `yaml:"maximum,omitempty" prop:"inner" alias:"max"`
ExclusiveMaximum any `yaml:"exclusiveMaximum,omitempty" prop:"inner" alias:"xmax"`
MultipleOf any `yaml:"multipleOf,omitempty" prop:"inner" alias:"mul"`
MinLength uint64 `yaml:"minLength,omitempty" prop:"inner" alias:"minlen"`
MaxLength *uint64 `yaml:"maxLength,omitempty" prop:"inner" alias:"maxlen"`
// properties below are applied to primitive or outer types
Description string `yaml:"description,omitempty" prop:"outer" alias:"desc,doc"`
Default any `yaml:"default,omitempty" prop:"outer" alias:"def"`
MinItems uint64 `yaml:"minItems,omitempty" prop:"outer" alias:"minsize"`
MaxItems *uint64 `yaml:"maxItems,omitempty" prop:"outer" alias:"maxsize"`
UniqueItems bool `yaml:"uniqueItems,omitempty" prop:"outer" alias:"unique"`
MinProperties uint64 `yaml:"minProperties,omitempty" prop:"outer" alias:"minprops"`
MaxProperties *uint64 `yaml:"maxProperties,omitempty" prop:"outer" alias:"maxprops"`
}
func (s *Schema) ApplyDefaults(source any) error {
return s.applyDefaults(reflect.ValueOf(source))
}
var errUnaddressable = errors.New(
`unable to set default value for unaddressable value (use pointer values if this is in a map or remove "default" tag)`)
func (s *Schema) applyDefaults(value reflect.Value) error {
if s.Default != nil && value.IsZero() {
if !value.CanAddr() {
return errUnaddressable
}
value.Set(reflect.ValueOf(s.Default))
return nil
}
if value.Kind() == reflect.Ptr {
if value.IsNil() {
if s.Default == nil {
return nil
}
if !value.CanAddr() {
return errUnaddressable
}
resolvedType := indirectType(value.Type())
value.Set(reflect.New(resolvedType))
}
value = value.Elem()
}
if schema, ok := s.AdditionalProperties.(*Schema); ok {
for _, key := range value.MapKeys() {
if err := schema.applyDefaults(value.MapIndex(key)); err != nil {
return errors.Wrapf(err, "on key %v", key.Interface())
}
}
return nil
}
if schema := s.Items; schema != nil {
for i := 0; i < value.Len(); i++ {
if err := schema.applyDefaults(value.Index(i)); err != nil {
return errors.Wrapf(err, "on index %d", i)
}
}
return nil
}
if properties := s.Properties; properties != nil {
for fieldNum := 0; fieldNum < value.NumField(); fieldNum++ {
field := value.Type().Field(fieldNum)
options := getYAMLOptions(field)
if options.inline {
if err := s.applyDefaults(value.Field(fieldNum)); err != nil {
return errors.Wrapf(err, "on embedded field %s", field.Name)
}
continue
}
property := properties[options.name]
if err := property.applyDefaults(value.Field(fieldNum)); err != nil {
return errors.Wrapf(err, "on field %s", options.name)
}
}
}
return nil
}
func GenerateSchema(value any) (*Schema, error) {
return makeSchema(reflect.TypeOf(value), "")
}
func makeSchema(valueType reflect.Type, tag reflect.StructTag) (*Schema, error) {
resolvedType := indirectType(valueType)
value := reflect.New(resolvedType).Elem()
sourceValue := value
if valueType.Kind() == reflect.Ptr {
sourceValue = sourceValue.Addr()
}
var node yaml.Node
if err := node.Encode(sourceValue.Interface()); err != nil {
return nil, errors.Wrap(err, "encode")
}
var elemType reflect.Type
var s Schema
switch {
case node.Tag == "!!str":
s.Type = "string"
switch value.Interface().(type) {
case time.Duration:
s.Pattern = `(\d+h)?(\d+m)?(\d+s)?(\d+ms)?(\d+µs)?(\d+ns)?`
default:
if formatted, ok := sourceValue.Interface().(formatted); ok {
s.Format = formatted.SchemaFormat()
}
if patterned, ok := sourceValue.Interface().(patterned); ok {
s.Pattern = patterned.SchemaPattern()
}
if enum, ok := sourceValue.Interface().(enum); ok {
s.Enum = enum.SchemaEnum()
}
}
case node.Tag == "!!timestamp":
s.Type = "string"
s.Format = "date-time"
case node.Tag == "!!int":
switch {
case value.CanInt(), value.CanUint():
s.Type = "integer"
default:
s.Type = "number"
}
case node.Tag == "!!float":
s.Type = "number"
case node.Tag == "!!bool":
s.Type = "boolean"
case node.Tag == "!!seq" && (resolvedType.Kind() == reflect.Slice || resolvedType.Kind() == reflect.Array):
elemType = valueType.Elem()
items, err := makeSchema(elemType, "")
if err != nil {
return nil, errors.Wrap(err, "generate items")
}
s.Type = "array"
s.Items = items
if resolvedType.Kind() == reflect.Array {
s.MinItems = uint64(value.Len())
s.MaxItems = pointer.To(uint64(value.Len()))
}
case node.Tag == "!!map" && resolvedType.Kind() == reflect.Map:
elemType = resolvedType.Elem()
additionalProperties, err := makeSchema(elemType, "")
if err != nil {
return nil, errors.Wrap(err, "generate additionalProperties")
}
s.Type = "object"
s.AdditionalProperties = additionalProperties
case node.Tag == "!!map" && resolvedType.Kind() == reflect.Struct:
properties, required, err := makeStructSchema(valueType)
if err != nil {
return nil, errors.Wrap(err, "generate properties & required")
}
s.Type = "object"
s.Properties = properties
s.AdditionalProperties = false
s.Required = required
}
if s.Type == "" {
return nil, errors.Errorf("unable to detect type for %s %s", node.Tag, resolvedType)
}
if err := applySchemaProps(&s, tag, valueType, elemType); err != nil {
return nil, errors.Wrap(err, "apply props")
}
return &s, nil
}
func applySchemaProps(s *Schema, tag reflect.StructTag, valueType, elemType reflect.Type) error {
schema := reflect.ValueOf(s).Elem()
schemaType := schema.Type()
for fieldNum := 0; fieldNum < schemaType.NumField(); fieldNum++ {
var (
field = schemaType.Field(fieldNum)
propType string
isArray bool
)
for i, option := range strings.Split(field.Tag.Get("prop"), ",") {
switch {
case i == 0:
propType = option
case option == "array":
isArray = true
}
}
if propType == "" {
continue
}
aliases := []string{getYAMLOptions(field).name}
aliases = append(aliases, strings.Split(field.Tag.Get("alias"), ",")...)
var prop string
for _, alias := range aliases {
if tag, ok := tag.Lookup(alias); ok {
prop = tag
break
}
}
if prop == "" {
continue
}
targetField := schema.Field(fieldNum)
var fieldValue reflect.Value
if field.Type.String() == "interface {}" {
targetType := valueType
if propType == "inner" && elemType != nil {
targetType = elemType
if s.Type == "array" {
targetField = reflect.Indirect(reflect.ValueOf(s.Items)).Field(fieldNum)
} else if s.Type == "object" && s.AdditionalProperties != false {
targetField = reflect.Indirect(reflect.ValueOf(s.AdditionalProperties)).Field(fieldNum)
}
}
if isArray {
prop = "[" + prop + "]"
fieldValue = reflect.New(reflect.SliceOf(targetType))
} else {
fieldValue = reflect.New(targetType)
}
} else {
fieldValue = reflect.New(field.Type)
}
if err := yaml.Unmarshal([]byte(prop), fieldValue.Interface()); err != nil {
return errors.Wrapf(err, "unmarshal %s", prop)
}
fieldValue = reflect.Indirect(fieldValue)
targetField.Set(fieldValue)
}
return nil
}
func makeStructSchema(valueType reflect.Type) (map[string]Schema, []string, error) {
var (
resolvedType = indirectType(valueType)
properties = make(map[string]Schema)
required []string
)
for fieldNum := 0; fieldNum < resolvedType.NumField(); fieldNum++ {
field := resolvedType.Field(fieldNum)
if !field.IsExported() {
continue
}
options := getYAMLOptions(field)
if options.inline {
embedded, err := makeSchema(field.Type, "")
if err != nil {
return nil, nil, errors.Wrapf(err, "generate embedded schema for %s", options.name)
}
required = append(required, embedded.Required...)
for name, property := range embedded.Properties {
properties[name] = property
}
continue
}
if !options.omitempty {
required = append(required, options.name)
}
property, err := makeSchema(resolvedType.Field(fieldNum).Type, field.Tag)
if err != nil {
return nil, nil, errors.Wrapf(err, "generate schema for %s", options.name)
}
properties[options.name] = *property
}
return properties, required, nil
}
type yamlOptions struct {
name string
omitempty bool
inline bool
}
func getYAMLOptions(field reflect.StructField) (options yamlOptions) {
for i, option := range strings.Split(field.Tag.Get("yaml"), ",") {
switch {
case i == 0:
if len(option) > 0 {
options.name = option
} else {
options.name = field.Name
}
case option == "omitempty":
options.omitempty = true
case option == "inline":
options.inline = true
}
}
return
}
func indirectType(typ reflect.Type) reflect.Type {
if typ.Kind() == reflect.Ptr {
typ = typ.Elem()
}
return typ
}
package confi
import (
"context"
"github.com/pkg/errors"
)
type Source interface {
GetValues(ctx context.Context) (map[string]any, error)
}
type PropertySource []Property
func (s PropertySource) GetValues(ctx context.Context) (map[string]any, error) {
values := make(map[string]any)
for _, prop := range s {
last := len(prop.Path) - 1
container := values
for i, el := range prop.Path {
if i == last {
if existing, ok := container[el].(map[string]any); existing == nil || !ok {
container[el] = prop.Value
}
continue
}
value := container[el]
if _, ok := value.(map[string]any); !ok {
container[el] = make(map[string]any)
}
container, _ = container[el].(map[string]any)
}
}
return values, nil
}
type InputSource struct {
Input Input
Format string
}
func (s InputSource) GetValues(ctx context.Context) (map[string]any, error) {
reader, err := s.Input.Reader()
if err != nil {
return nil, errors.Wrap(err, "open input")
}
defer CloseQuietly(reader)
if s.Format == "properties" {
props, err := readProperties(reader)
if err != nil {
return nil, errors.Wrap(err, "read properties")
}
return PropertySource(props).GetValues(ctx)
}
codec, ok := Codecs[s.Format]
if !ok {
return nil, errors.Wrapf(err, "no codec found for %s", s.Format)
}
values := make(map[string]any)
if err := codec.UnmarshalFn(reader, &values); err != nil {
return nil, err
}
return values, nil
}
package confi
import (
"context"
"io"
"path/filepath"
"strings"
"github.com/pkg/errors"
)
type SourceProvider interface {
GetSources(ctx context.Context) ([]Source, error)
}
type DefaultSourceProvider struct {
EnvPrefix string
Env []string
Args []string
Stdin io.Reader
}
func (p *DefaultSourceProvider) GetSources(ctx context.Context) ([]Source, error) {
var envProps, argProps []Property
for _, env := range p.Env {
if strings.HasPrefix(env, p.EnvPrefix) {
env = env[len(p.EnvPrefix):]
prop, err := getProperty(env, "_", false)
if err != nil {
return nil, errors.Wrapf(err, `env "%s"`, env)
}
envProps = append(envProps, *prop)
}
}
for _, arg := range p.Args {
if strings.HasPrefix(arg, "--") {
arg = arg[2:]
prop, err := getProperty(arg, ".", true)
if err != nil {
return nil, errors.Wrapf(err, `arg "%s"`, arg)
}
argProps = append(argProps, *prop)
}
}
var (
envs PropertySource
files []Source
stdin *InputSource
args PropertySource
)
for _, item := range []struct {
source *PropertySource
props []Property
}{
{source: &envs, props: envProps},
{source: &args, props: argProps},
} {
hasFiles := false
for _, prop := range item.props {
switch prop.Key() {
case "config.file":
if !hasFiles {
files = make([]Source, 0)
}
format := filepath.Ext(prop.Value)
if format == "" {
continue
}
files = append(files, InputSource{
Input: File(prop.Value),
Format: format[1:],
})
hasFiles = true
case "config.stdin":
if prop.Value == "" {
stdin = nil
} else {
stdin = &InputSource{
Input: Reader{R: p.Stdin},
Format: prop.Value,
}
}
default:
*item.source = append(*item.source, prop)
}
}
}
sources := make([]Source, 0)
if envs != nil {
sources = append(sources, envs)
}
sources = append(sources, files...)
if stdin != nil {
sources = append(sources, *stdin)
}
if args != nil {
sources = append(sources, args)
}
return sources, nil
}