omega
install
go get github.com/hotfizz/omega@latest
go install github.com/hotfizz/omega@latest
examples
package main
import (
"fmt"
"os"
"github.com/hotfizz/omega/alt"
)
func objectHelper() (objects []interface{}) {
objects = append(objects, 1)
objects = append(objects, 1, "foo")
objects = append(objects, 1, "foo", "baz")
objects = append(objects, 1, "foo", "baz", map[string]interface{}{"key": "base"})
objects = append(objects, 1, "foo", "baz",
map[string]interface{}{
"key": "base",
"list": []interface{}{1, "99"},
})
return objects
}
func main() {
objects := objectHelper()
parse := alt.NewDataEtlParser(
alt.SetIgnore(map[string]struct{}{}), // set ignore (key) list
// set logger, level is debug, output device is console
alt.SetLogger(alt.NewStdLogger(alt.LevelDebug, os.Stdout)),
alt.SetMaxDepth(alt.Infinity), // infinity depth, warn stack
alt.SetSeparator(alt.StrSeparator("_")), // use underline as separator
)
for _, obj := range objects {
result := parse.Parse(obj)
fmt.Println(result)
}
}
用于将对象扁平化输出的库
常见的场景: 比如 elastic, mongo,通用 🕷 API 返回的 JSON 数据转为有结构化的数据
特别是将数仓 ETL 过程中,将嵌套的数据结构 保存到结构化存储引擎中
比如, 下面这个例子:
{
"name": "map",
"data": [
{
"user_name": "小明",
"age": 18,
"province": "广东"
},
{
"user_name": "小海",
"age": 17,
"province": "海南"
}
],
"data2": [
{
"persons": [
{
"address": "广东"
},
{
"address": "海南"
}
]
}
]
}
这样的数据是没有办法入库的(clickhouse, doris, mysql) 这种需要预先定义结构的数据库,所以,需要将这种半结构化的数据转为结构化的
上面的数据会解析成下面这样(一共四条)
[
{
"data.age": 18,
"data.province": "广东",
"data.user_name": "小明",
"data2.persons.address": "广东",
"name": "map"
},
{
"data.age": 18,
"data.province": "广东",
"data.user_name": "小明",
"data2.persons.address": "海南",
"name": "map"
},
{
"data.age": 17,
"data.province": "海南",
"data.user_name": "小海",
"data2.persons.address": "广东",
"name": "map"
},
{
"data.age": 17,
"data.province": "海南",
"data.user_name": "小海",
"data2.persons.address": "海南",
"name": "map"
}
]
只有转成这种结构之后,后面的处理程序才能更好地将这四条数据入库
具体的实现