cmd

Documentation

Index

• Variables
• func Execute()
• func ReplaceSoloCarriageReturns(data io.Reader) io.Reader
• type CatalogMap
• type Feature
• type OpenFoodFact
• type Product

Variables

var PrepareCmd = &cobra.Command{
	Use:     "prepare",
	Aliases: []string{"p"},
	Short:   "prepare csv catalog to prestashop import format (core or webkul's marketplace).",
	Long:    "prepare csv catalog to prestashop import format (core or webkul's marketplace).",
	Run: func(cmd *cobra.Command, args []string) {

		gtotal := 0

		if !dryRun {
			var err error

			dsn := fmt.Sprintf("%v:%v@tcp(%v:%v)/%v?charset=utf8mb4&collation=utf8mb4_general_ci&parseTime=True&loc=Local", dbUser, dbPass, dbHost, dbPort, dbName)
			db, err = gorm.Open(mysql.Open(dsn), &gorm.Config{
				NamingStrategy: schema.NamingStrategy{
					TablePrefix:   dbTablePrefix,
					SingularTable: true,
					NameReplacer:  strings.NewReplacer("ID", "Id"),
				},
			})
			if err != nil {
				log.Fatal(err)
			}

		}

		var err error
		proxyURL, err = url.Parse(proxyURLStr)
		if err != nil {
			log.Fatal(err)
		}

		kv, err = badger.Open(badger.DefaultOptions(kvPath))
		if err != nil {
			log.Fatal(err)
		}
		defer kv.Close()

		var catalogs []string
		reStruct := regexp.MustCompile(`type (.*) struct`)
		reTags := regexp.MustCompile(`json:"(.*)"`)

		err = db.Where("active = ?", 1).Find(&activeLangs).Error
		pp.Println("activeLangs:", activeLangs)

		err = db.Where("active = ?", 1).Find(&activeShops).Error
		if errors.Is(err, gorm.ErrRecordNotFound) {
			log.Fatal("active shops not found")
		}
		pp.Println("activeShops:", activeShops)

		err = db.Find(&activeGroups).Error
		if errors.Is(err, gorm.ErrRecordNotFound) {
			log.Fatal("groups not found")
		}
		pp.Println("activeGroups:", activeGroups)

		feedURL := "https://static.openfoodfacts.org/data/en.openfoodfacts.org.products.csv"
		feedName := "OpenFoodFact"

		pp.Println("feedName:", feedName)
		pp.Println("feedURL:", feedURL)

		cmp := &CatalogMap{
			Name: feedName,
			Feed: feedURL,
		}

		cmp.Mapping.LangSuffix = languagesDef
		cmp.Mapping.LangFields = []string{"name", "description", "description_short"}

		client := grab.NewClient()

		tr := &http.Transport{

			DialContext: (&net.Dialer{
				Timeout: 240 * time.Second,

				DualStack: true,
			}).DialContext,
			MaxIdleConns:          100,
			IdleConnTimeout:       240 * time.Second,
			TLSHandshakeTimeout:   240 * time.Second,
			ExpectContinueTimeout: 1 * time.Second,
			DisableKeepAlives:     true,
			TLSClientConfig: &tls.Config{
				InsecureSkipVerify: true,
			},
		}

		client.UserAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.97 Safari/537.36"
		client.HTTPClient = &http.Client{
			Timeout:   480 * time.Second,
			Transport: tr,
			CheckRedirect: func(req *http.Request, via []*http.Request) error {
				return nil
			},
		}

		feedName = stripCtlAndExtFromUnicode(feedName)

		localCatalogStruct := filepath.Join("..", "..", "internal", "openfoodfact", fmt.Sprintf("%s.go", strcase.ToSnake(feedName)))
		localCatalogJson := filepath.Join("catalogs", fmt.Sprintf("%s.json", slug.Make(feedName)))
		localCatalogCsv := filepath.Join("catalogs", fmt.Sprintf("%s.csv", slug.Make(feedName)))
		localCatalogMapYaml := filepath.Join("imports", fmt.Sprintf("%s.yml", slug.Make(feedName)))
		localCatalogFormattedCsv := filepath.Join("imports", fmt.Sprintf("%s.csv", slug.Make(feedName)))

		if _, err := os.Stat(localCatalogCsv); os.IsNotExist(err) {

			req, err := grab.NewRequest(localCatalogCsv, feedURL)
			if err != nil {
				checkErr("grab.NewRequest.error", err)
			}

			fmt.Printf("Downloading %v...\n", req.URL())
			resp := client.Do(req)
			if resp.HTTPResponse == nil {
				checkErr("grab.HTTPResponse.error", err)
			}
			fmt.Printf("  %v\n", resp.HTTPResponse.Status)

			t := time.NewTicker(500 * time.Millisecond)
			defer t.Stop()

		LoopRel:
			for {
				select {
				case <-t.C:
					fmt.Printf("  transferred %v / %v bytes (%.2f%%)\n",
						resp.BytesComplete(),
						resp.Size,
						100*resp.Progress())

				case <-resp.Done:

					break LoopRel
				}
			}

			if err := resp.Err(); err != nil {
				checkErr("grab.Download.error", err)
			}

			fmt.Printf("Download saved to %v \n", resp.Filename)
			localCatalogCsv = resp.Filename
		}

		inputFile, err := os.Open(localCatalogCsv)
		if err != nil {
			checkErr("localCatalogCsv.error", err)
		}
		defer inputFile.Close()

		cmp.Separator = "\t"

		cols, err := getHeaders(localCatalogCsv, "\t")
		if err != nil {
			checkErr("getHeaders.error", err)
		}

		cmp.Fields = cols
		cmp.Mapping.Update = true

		for _, col := range cols {
			switch col {
			case "url":
				cmp.Mapping.Product.Redirect = col
			case "product_name":
				cmp.Mapping.Product.Name = col
			case "code":
				cmp.Mapping.Product.Reference = col

			case "packaging_text":
				cmp.Mapping.Product.Description = col
			case "ingredients_text":
				cmp.Mapping.Product.DescriptionShort = col
			case "quantity":
				cmp.Mapping.Product.Quantity = col
			case "image_url":
				cmp.Mapping.Product.Image = col

			case "brands", "packaging", "origins", "manufacturing_places", "labels", "emb_codes", "first_packaging_code_geo", "cities", "purchase_places", "stores", "countries", "allergens", "traces", "serving_size", "serving_quantity", "no_nutriments", "additives_n", "additives", "additives_tags", "additives_eningredients_from_palm_oil_n", "ingredients_from_palm_oil", "ingredients_from_palm_oil_tags", "ingredients_that_may_be_from_palm_oil_n", "ingredients_that_may_be_from_palm_oil", "ingredients_that_may_be_from_palm_oil_tags", "nutriscore_score", "nutriscore_grade", "nova_group", "pnns_groups_1", "pnns_groups_2", "brand_owner", "energy-kj_100g", "energy-kcal_100g", "energy_100g", "energy-from-fat_100g", "fat_100g", "saturated-fat_100g", "-butyric-acid_100g", "-caproic-acid_100g", "-caprylic-acid_100g", "-capric-acid_100g", "-lauric-acid_100g", "-myristic-acid_100g-palmitic-acid_100g", "-stearic-acid_100g", "-arachidic-acid_100g", "-behenic-acid_100g", "-lignoceric-acid_100g", "-cerotic-acid_100g", "-montanic-acid_100g", "-melissic-acid_100g", "monounsaturated-fat_100g", "polyunsaturated-fat_100g", "omega-3-fat_100g", "-alpha-linolenic-acid_100g", "-eicosapentaenoic-acid_100g", "-docosahexaenoic-acid_100g", "omega-6-fat_100g", "-linoleic-acid_100g", "-arachidonic-acid_100g", "-gamma-linolenic-acid_100g", "-dihomo-gamma-linolenic-acid_100g", "omega-9-fat_100g", "-oleic-acid_100g", "-elaidic-acid_100g", "-gondoic-acid_100g", "-mead-acid_100g", "-erucic-acid_100g", "-nervonic-acid_100g", "trans-fat_100g", "cholesterol_100g", "carbohydrates_100g", "sugars_100g", "-sucrose_100g", "-glucose_100g", "-fructose_100g", "-lactose_100g", "-maltose_100g", "-maltodextrins_100g", "starch_100g", "polyols_100g", "fiber_100g", "-soluble-fiber_100g", "-insoluble-fiber_100g", "proteins_100g", "casein_100g", "serum-proteins_100g", "nucleotides_100g", "salt_100g", "sodium_100g", "alcohol_100g", "vitamin-a_100g", "beta-carotene_100g", "vitamin-d_100g", "vitamin-e_100g", "vitamin-k_100g", "vitamin-c_100g", "vitamin-b1_100g", "vitamin-b2_100g", "vitamin-pp_100g", "vitamin-b6_100g", "vitamin-b9_100g", "folates_100g", "vitamin-b12_100g", "biotin_100g", "pantothenic-acid_100g", "silica_100g", "bicarbonate_100g", "potassium_100g", "chloride_100g", "calcium_100g", "phosphorus_100g", "iron_100g", "magnesium_100g", "zinc_100g", "copper_100g", "manganese_100g", "fluoride_100g", "selenium_100g", "chromium_100g", "molybdenum_100g", "iodine_100g", "caffeine_100g", "taurine_100g", "ph_100g", "fruits-vegetables-nuts_100g", "fruits-vegetables-nuts-dried_100g", "fruits-vegetables-nuts-estimate_100g", "collagen-meat-protein-ratio_100g", "cocoa_100g", "chlorophyl_100g", "carbon-footprint_100g", "carbon-footprint-from-meat-or-fish_100g", "nutrition-score-fr_100g", "nutrition-score-uk_100g", "glycemic-index_100g", "water-hardness_100g", "choline_100g", "phylloquinone_100g", "beta-glucan_100g", "inositol_100g", "carnitine_100g":
				cmp.Mapping.Product.Features = append(cmp.Mapping.Product.Features, col)
			case "main_category":
				cmp.Mapping.Category.Name = col
			case "categories", "categories_en":
				cmp.Mapping.Category.Breadcrumb = col
			}
		}

		cmp, total, err := csv2ps(db, localCatalogCsv, localCatalogFormattedCsv, cols, "\t", cmp)
		checkErr("while writing formatted for prestashop csv file", err)

		cmp.Total = total
		gtotal += total

		cmpBytes, err := yaml.Marshal(cmp)
		if err != nil {
			checkErr("yaml.Marshal.Error:", err)
		}
		err = ioutil.WriteFile(localCatalogMapYaml, cmpBytes, 0644)
		checkErr("while writing mapping yaml file", err)

		os.Exit(1)

		json2struct.SetDebug(options.debug)
		opt := json2struct.Options{
			UseOmitempty:   false,
			UseShortStruct: true,
			UseLocal:       false,
			UseExample:     false,
			Prefix:         "",
			Suffix:         "",
			Name:           strings.ToLower(feedName),
		}

		jsonFile, err := os.Open(localCatalogJson)
		if err != nil {
			checkErr("localCatalogJson.Error:", err)
		}
		defer jsonFile.Close()

		parsed, err := json2struct.Parse(jsonFile, opt)
		if err != nil {
			checkErr("json2struct.Error", err)
		}

		csvRowLine := 1
		csvRowExample, err := getExampleRow(localCatalogCsv, "\t", cols, csvRowLine)

		if err != nil {
			checkErr("getExampleRow.Error", err)
		}

		structName := reStruct.FindStringSubmatch(parsed)

		parsed = reTags.ReplaceAllString(parsed, `json:"$1" struct2map:"key:$1"`)

		catalogs = append(catalogs, structName[1])

		structResult := bytes.NewBufferString("")
		structTemplate, _ := template.New("").Parse(packageTemplate)
		structTemplate.Execute(structResult, map[string]string{
			"CatalogSeparator": "\t",
			"CatalogPath":      localCatalogCsv,
			"Line":             fmt.Sprintf("%d", csvRowLine),
			"Struct":           parsed,
			"StructName":       stripCtlAndExtFromUnicode(structName[1]),
			"Row":              strings.Join(csvRowExample, ",\n")},
		)

		err = ioutil.WriteFile(localCatalogStruct, structResult.Bytes(), 0644)

		if err != nil {
			checkErr("localCatalogStruct.Error", err)
		}

		localCatalogBase := filepath.Join("..", "..", "internal", "openfoodfact", fmt.Sprintf("%s.go", "base"))
		catalogResult := bytes.NewBufferString("")
		catalogTemplate, _ := template.New("").Parse(baseTemplate)
		catalogTemplate.Execute(catalogResult, map[string][]string{"Catalogs": catalogs})
		err = ioutil.WriteFile(localCatalogBase, catalogResult.Bytes(), 0644)
		checkErr("while writing struct file", err)

		pp.Println("total new entries:", gtotal)

	},
}

var RootCmd = &cobra.Command{
	Use:   "off2ps",
	Short: "off2ps is an helper to load OpenFoodFact csv catalog into a prestashop database.",
	Long:  `off2ps is an helper to load OpenFoodFact csv catalog into a prestashop database.`,
}

RootCmd is the root command for ovh-qa

var Tables = []interface{}{
	&OpenFoodFact{},
}

Functions

func Execute

func Execute()

Execute adds all child commands to the root command and sets flags appropriately.

func ReplaceSoloCarriageReturns

func ReplaceSoloCarriageReturns(data io.Reader) io.Reader

ReplaceSoloCarriageReturns wraps an io.Reader, on every call of Read it for instances of lonely \r replacing them with \r\n before returning to the end customer lots of files in the wild will come without "proper" line breaks, which irritates go's standard csv package. This'll fix by wrapping the reader passed to csv.NewReader:

rdr, err := csv.NewReader(ReplaceSoloCarriageReturns(r))

Types

type CatalogMap

type CatalogMap struct {
	Name      string   `yaml:"name"`
	Feed      string   `yaml:"feed"`
	SellerId  int      `yaml:"seller_id"`
	Separator string   `yaml:"separator"`
	Fields    []string `yaml:"fields"`
	Total     int      `yaml:"total"`
	Mapping   struct {
		Update     bool     `yaml:"update" default:"true"` // Either ADD
		LangSuffix []string `yaml:"multi_language_suffix"`
		LangFields []string `yaml:"multi_language_fields"`
		Product    Product  `yaml:"product"`
		Category   struct {
			Name       string `yaml:"name"`
			Breadcrumb string `yaml:"breaddcrumb"`
			Separator  string `yaml:"separator"`
		} `yaml:"category"`
	} `yaml:"mapping"`
}

type Feature

type Feature struct {
	IDFeature      uint
	IDFeatureValue uint
}

type OpenFoodFact

type OpenFoodFact struct {
	Additives                string         `json:"additives,omitempty" csv:"additives,omitempty"`
	AdditivesN               string         `json:"additives_n,omitempty" csv:"additives_n,omitempty"`
	AdditivesTags            string         `json:"additives_tags,omitempty" csv:"additives_tags,omitempty"`
	Allergens                string         `json:"allergens,omitempty" csv:"allergens,omitempty"`
	AllergensEn              string         `json:"allergens_en,omitempty" csv:"allergens_en,omitempty"`
	AbbreviatedProductName   string         `json:"abbreviated_product_name,omitempty" csv:"abbreviated_product_name,omitempty"`
	AdditivesEn              string         `json:"additives_en,omitempty" csv:"additives_en,omitempty"`
	BrandOwner               string         `json:"brand_owner,omitempty" csv:"brand_owner,omitempty"`
	Brands                   string         `json:"brands,omitempty" csv:"brands,omitempty"`
	BrandsTags               string         `json:"brands_tags,omitempty" csv:"brands_tags,omitempty"`
	Categories               string         `json:"categories,omitempty" csv:"categories,omitempty"`
	CategoriesEn             string         `json:"categories_en,omitempty" csv:"categories_en,omitempty"`
	CategoriesTags           string         `json:"categories_tags,omitempty" csv:"categories_tags,omitempty"`
	Cities                   string         `json:"cities,omitempty" csv:"cities,omitempty"`
	CitiesTags               string         `json:"cities_tags,omitempty" csv:"cities_tags,omitempty"`
	Countries                string         `json:"countries,omitempty" csv:"countries,omitempty"`
	CountriesEn              string         `json:"countries_en,omitempty" csv:"countries_en,omitempty"`
	CountriesTags            string         `json:"countries_tags,omitempty" csv:"countries_tags,omitempty"`
	CreatedDatetime          datatypes.Date `json:"created_datetime,omitempty" csv:"created_datetime,omitempty"`
	CreatedT                 time.Time      `json:"created_t,omitempty" csv:"created_t,omitempty"`
	Creator                  string         `json:"creator,omitempty" csv:"creator,omitempty"`
	EmbCodes                 string         `json:"emb_codes,omitempty" csv:"emb_codes,omitempty"`
	EmbCodesTags             string         `json:"emb_codes_tags,omitempty" csv:"emb_codes_tags,omitempty"`
	Code                     string         `gorm:"uniqueIndex:code_idx;" json:"code,omitempty" csv:"code,omitempty"`
	GenericName              string         `json:"generic_name,omitempty" csv:"generic_name,omitempty"`
	ImageIngredientsSmallURL string         `json:"image_ingredients_small_url,omitempty" csv:"image_ingredients_small_url,omitempty"`
	ImageIngredientsURL      string         `json:"image_ingredients_url,omitempty" csv:"image_ingredients_url,omitempty"`
	ImageNutritionSmallURL   string         `json:"image_nutrition_small_url,omitempty" csv:"image_nutrition_small_url,omitempty"`
	ImageNutritionURL        string         `json:"image_nutrition_url,omitempty" csv:"image_nutrition_url,omitempty"`
	ImageSmallURL            string         `json:"image_small_url,omitempty" csv:"image_small_url,omitempty"`
	ImageURL                 string         `json:"image_url,omitempty" csv:"image_url,omitempty"`
	Labels                   string         `json:"labels,omitempty" csv:"labels,omitempty"`
	LabelsEn                 string         `json:"labels_en,omitempty" csv:"labels_en,omitempty"`
	LabelsTags               string         `json:"labels_tags,omitempty" csv:"labels_tags,omitempty"`
	LastModifiedDatetime     time.Time      `json:"last_modified_datetime,omitempty" csv:"last_modified_datetime,omitempty"`
	LastModifiedT            time.Time      `json:"last_modified_t,omitempty" csv:"last_modified_t,omitempty"`
	NoNutriments             string         `json:"no_nutriments,omitempty" csv:"no_nutriments,omitempty"`
	NovaGroup                string         `json:"nova_group,omitempty" csv:"nova_group,omitempty"`
	MainCategory             string         `json:"main_category,omitempty" csv:"main_category,omitempty"`
	MainCategoryEn           string         `json:"main_category_en,omitempty" csv:"main_category_en,omitempty"`
	PurchasePlaces           string         `json:"purchase_places,omitempty" csv:"purchase_places,omitempty"`
	ManufacturingPlaces      string         `json:"manufacturing_places,omitempty" csv:"manufacturing_places,omitempty"`
	ManufacturingPlacesTags  string         `json:"manufacturing_places_tags,omitempty" csv:"manufacturing_places_tags,omitempty"`
	NutriscoreGrade          string         `json:"nutriscore_grade,omitempty" csv:"nutriscore_grade,omitempty"`
	NutriscoreScore          string         `json:"nutriscore_score,omitempty" csv:"nutriscore_score,omitempty"`
	Origins                  string         `json:"origins,omitempty" csv:"origins,omitempty"`
	OriginsEn                string         `json:"origins_en,omitempty" csv:"origins_en,omitempty"`
	OriginsTags              string         `json:"origins_tags,omitempty" csv:"origins_tags,omitempty"`
	Packaging                string         `json:"packaging,omitempty" csv:"packaging,omitempty"`
	PackagingTags            string         `json:"packaging_tags,omitempty" csv:"packaging_tags,omitempty"`
	PackagingText            string         `json:"packaging_text,omitempty" csv:"packaging_text,omitempty"`
	ProductName              string         `json:"product_name,omitempty" csv:"product_name,omitempty"`
	Quantity                 string         `json:"quantity,omitempty" csv:"quantity,omitempty"`
	ServingQuantity          string         `json:"serving_quantity,omitempty" csv:"serving_quantity,omitempty"`
	ServingSize              string         `json:"serving_size,omitempty" csv:"serving_size,omitempty"`
	States                   string         `json:"states,omitempty" csv:"states,omitempty"`
	StatesEn                 string         `json:"states_en,omitempty" csv:"states_en,omitempty"`
	StatesTags               string         `json:"states_tags,omitempty" csv:"states_tags,omitempty"`
	Stores                   string         `json:"stores,omitempty" csv:"stores,omitempty"`
	URL                      string         `json:"url,omitempty" csv:"url,omitempty"`
	Traces                   string         `json:"traces,omitempty" csv:"traces,omitempty"`
	TracesEn                 string         `json:"traces_en,omitempty" csv:"traces_en,omitempty"`
	TracesTags               string         `json:"traces_tags,omitempty" csv:"traces_tags,omitempty"`
}

type Product

type Product struct {
	Name             string `yaml:"name"`
	Reference        string `yaml:"reference"`
	Price            string `yaml:"price"`
	Description      string `yaml:"description"`
	DescriptionShort string `yaml:"description_short"`
	Image            string `yaml:"image"`
	Quantity         string `yaml:"quantity"`
	Redirect         string `yaml:"redirect"`
	// Attributes  []string `yaml:"attributes"`
	Features []string `yaml:"features"`
}