index

Documentation

Index

• Variables
• type IndexMakeEvent
• type RepoDoc
• type RepoInfo
• type RepoPath
• type ReposetRef
• type ReposetRefList

Variables

var AddDocumentCmd = &cmds.Command{
	Helptext: cmds.HelpText{
		Tagline: "Add document to index repository.",
		ShortDescription: `
Add a document to index repository specified by path.
`,
		LongDescription: `
Add a document to an index repository at path containing content
of a similar kind.

Use the create document command to create an empty document template
with all the fields pre-generated. After editing the document to write
the desired content, add the document to the repository.

	dms3 index mkdoc -k="blog" --xml > b.xml # edit document
	dms3 index addoc b.xml <path>          # add blog to reposet

Use --xml option to convey repository input document format.

`,
	},

	Arguments: []cmds.Argument{
		cmds.StringArg("file", true, false, "content to add to repository."),
		cmds.StringArg("dms3-path", true, false, "path to repository."),
	},
	Options: []cmds.Option{
		cmds.BoolOption(addQuietOptionName, "q", "Write just hashes of created object.").WithDefault(true),
	},
	Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error {
		var file, repo string
		if len(req.Arguments) != 2 {
			msg := fmt.Sprintf("file and path are both required.\n.")
			errmsg := errors.New(msg)

			return errmsg
		}
		file = req.Arguments[0]
		repo = req.Arguments[1]

		log.Debugf("file value is %s, repo path is %s", file, repo)

		n, err := cmdenv.GetNode(env)
		if err != nil {
			return err
		}

		log.Debugf("Running command request path %s", req.Path)

		adopts := new(addocOpts)

		q, _ := req.Options[addQuietOptionName].(bool)
		if q {
			adopts.q = true
		} else {
			adopts.q = false
		}
		log.Debugf("quiet option value %t", q)

		ctx := req.Context

		/*
			p, err := path.ParsePath(p)
			if err != nil {
				return err
			}
		*/
		var p path.Path

		output, err := addDoc(ctx, n, p, adopts)
		if err != nil {
			return err
		}

		logOutputValue(output)

		return cmds.EmitOnce(res, output)
	},
	Encoders: cmds.EncoderMap{
		cmds.Text: cmds.MakeEncoder(func(req *cmds.Request, w io.Writer, v interface{}) error {
			repoPath, ok := v.(*RepoPath)
			if !ok {
				return e.TypeErr(repoPath, v)
			}

			_, err := fmt.Fprintf(w, "path %s \n", repoPath.path)
			return err
		}),
	},
	Type: RepoPath{},
}

AddDocumentCmd add document to index

var ErrDepthLimitExceeded = fmt.Errorf("depth limit exceeded")

ErrDepthLimitExceeded indicates that the max depth has been exceeded.

var ErrNotYetImplemented = fmt.Errorf("not yet implemented")

ErrNotYetImplemented indicates that the max depth has been exceeded.

var ListIndexCmd = &cmds.Command{
	Helptext: cmds.HelpText{
		Tagline: "List index repositories.",
		ShortDescription: `
Returns the list of local index repositories.
`,
		LongDescription: `
Returns the list of local index repositories.
By default, all repositories are returned.

Use the '--kind' flag to match a specific repository kind.
Use the '--name' flag to match a specific repository name.
Use the '--meta' flag to list only metastore repositories.
Use the '--info' flag to list only infostore repositories.
Use the '--offset' flag to specify result starting page offset.
Use the '--length' flag to specify length of each result page.
`,
	},

	Arguments: []cmds.Argument{},
	Options: []cmds.Option{
		cmds.StringOption(lsKindOptionName, "k", "Kind of repository to list."),
		cmds.StringOption(lsNameOptionName, "n", "Name of repository to list."),
		cmds.BoolOption(lsMetaOptionName, "m", "List metastore repositories.").WithDefault(true),
		cmds.BoolOption(lsDataOptionName, "d", "List infostore repositories.").WithDefault(true),
		cmds.IntOption(lsOffsetOptionName, "p", "Page offset.").WithDefault(0),
		cmds.IntOption(lsLengthOptionName, "l", "Page length.").WithDefault(24),
	},
	Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error {
		kopt, _ := req.Options[lsKindOptionName].(string)
		nopt, _ := req.Options[lsNameOptionName].(string)
		mopt, _ := req.Options[lsMetaOptionName].(bool)
		dopt, _ := req.Options[lsDataOptionName].(bool)
		popt, _ := req.Options[lsOffsetOptionName].(int)
		lopt, _ := req.Options[lsLengthOptionName].(int)
		log.Debugf("kind option %s", kopt)
		log.Debugf("name option %s", nopt)
		log.Debugf("meta option %v", mopt)
		log.Debugf("info option %v", dopt)
		log.Debugf("offset option %v", popt)
		log.Debugf("length option %v", lopt)

		n, err := cmdenv.GetNode(env)
		if err != nil {
			return err
		}

		_, _ = listLocalRepo(n, kopt, nopt, mopt, dopt, popt, lopt)

		return nil
	},
	Encoders: cmds.EncoderMap{
		cmds.Text: cmds.MakeEncoder(func(req *cmds.Request, w io.Writer, v interface{}) error {
			list, ok := v.(ReposetRefList)
			if !ok {
				return e.TypeErr(list, v)
			}

			for i := range list {
				if _, err := fmt.Fprintf(w, "%v %v %v %v\n", list[i].Infoclass, list[i].Reposetkind, list[i].Reposetname, list[i].Reposetpath); err != nil {
					return err
				}
			}
			return nil
		}),
	},
	Type: ReposetRefList{},
}

ListIndexCmd list index repositories

var ListPubIndexCmd = &cmds.Command{
	Helptext: cmds.HelpText{
		Tagline: "List index repositories.",
		ShortDescription: `
Returns the list of local index repositories.
`,
		LongDescription: `
Returns the list of local index repositories.
By default, all repositories are returned.

Use the '--kind' flag to match a specific repository kind.
Use the '--name' flag to match a specific repository name.
Use the '--meta' flag to list metastore repositories.
Use the '--data' flag to list infostore repositories.
Use the '--page-offset' flag to specify result starting page offset.
Use the '--page-length' flag to specify length of each result page.
`,
	},

	Arguments: []cmds.Argument{},
	Options: []cmds.Option{
		cmds.StringOption(lsKindOptionName, "k", "Kind of repository to list."),
		cmds.StringOption(lsNameOptionName, "n", "Name of repository to list."),
		cmds.BoolOption(lsMetaOptionName, "m", "List metastore repositories.").WithDefault(true),
		cmds.BoolOption(lsDataOptionName, "d", "List infostore repositories.").WithDefault(true),
		cmds.IntOption(lsOffsetOptionName, "p", "Page offset.").WithDefault(0),
		cmds.IntOption(lsLengthOptionName, "l", "Page length.").WithDefault(24),
	},
	Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error {
		kopt, _ := req.Options[lsKindOptionName].(string)
		nopt, _ := req.Options[lsNameOptionName].(string)
		mopt, _ := req.Options[lsMetaOptionName].(bool)
		dopt, _ := req.Options[lsDataOptionName].(bool)
		popt, _ := req.Options[lsOffsetOptionName].(int)
		lopt, _ := req.Options[lsLengthOptionName].(int)
		log.Debugf("kind option %s", kopt)
		log.Debugf("name option %s", nopt)
		log.Debugf("meta option %v", mopt)
		log.Debugf("data option %v", dopt)
		log.Debugf("offset option %v", popt)
		log.Debugf("length option %v", lopt)

		n, err := cmdenv.GetNode(env)
		if err != nil {
			return err
		}

		output, err := listRepo(n, kopt, nopt, mopt, dopt, popt, lopt)
		if err != nil {
			return err
		}

		logOutputValue(output)

		return cmds.EmitOnce(res, output)
	},
	Encoders: cmds.EncoderMap{
		cmds.Text: cmds.MakeEncoder(func(req *cmds.Request, w io.Writer, v interface{}) error {
			list, ok := v.(ReposetRefList)
			if !ok {
				return e.TypeErr(list, v)
			}

			for i := range list {
				if _, err := fmt.Fprintf(w, "%v %v %v %v\n", list[i].Infoclass, list[i].Reposetkind, list[i].Reposetname, list[i].Reposetpath); err != nil {
					return err
				}
			}
			return nil
		}),
	},
	Type: ReposetRefList{},
}

ListPubIndexCmd list published index repositories

var MakeDocumentCmd = &cmds.Command{
	Helptext: cmds.HelpText{
		Tagline: "Make document template for new content.",
		ShortDescription: `
Make an empty document for editing new content.
`,
		LongDescription: `
Make a document for editing new content of a similar kind.
The content kind is named using a locally unique key ex: blog.

Use the create document command to create an empty document template
with all the fields pre-generated.

	dms3 index mkdoc -k=blog > b.xml    # edit document, then
	dms3 index addoc b.xml <path>       # add blog to reposet

`,
	},

	Arguments: []cmds.Argument{},
	Options: []cmds.Option{
		cmds.StringOption("kind", "k", "keyword for kind of content, ex: \"blog\" ."),
	},
	Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error {
		n, err := cmdenv.GetNode(env)
		if err != nil {
			return err
		}

		kopt, _ := req.Options[kindOptionName].(string)
		if kopt == "" {
			return fmt.Errorf("kind of content key must be specified.")
		}
		log.Debugf("kind option value %s", kopt)

		icfg, err := n.Repo.IdxConfig()
		if err != nil {
			return fmt.Errorf("could not load index config.")
		}

		var repodoc *RepoDoc

		output, err := idxlfs.MakeDoc(*icfg, kopt)
		if err != nil {
			return err
		} else {
			repodoc = &RepoDoc{
				content: output,
			}
		}

		return cmds.EmitOnce(res, repodoc)
	},
	Encoders: cmds.EncoderMap{
		cmds.Text: cmds.MakeEncoder(func(req *cmds.Request, w io.Writer, v interface{}) error {
			repoDoc, ok := v.(*RepoDoc)
			if !ok {
				return e.TypeErr(repoDoc, v)
			}

			_, err := fmt.Fprintf(w, "%v\n", repoDoc.content)
			return err
		}),
	},
	Type: RepoDoc{},
}

var MakeIndexCmd = &cmds.Command{
	Helptext: cmds.HelpText{
		Tagline: "Make index repository set.",
		ShortDescription: `
Make a new searchable repository set.
`,
		LongDescription: `
Make a new searchable infostore or metastore for documents of
a similar kind. The repository kind is named using a locally
unique key ex: blog.

Each created repository set can be customized with specific schema
fields to expose structure of documents it will host. The exposed
structure can be used to refine search with the robust supported
query language.

The set of fields used for a specifc kind key can be customized
using the repository configure command.

	dms3 index config show    # to show index configuration
	dms3 index config --json Metadata.Kind \
		'[{"Name": "blog", "Field": ["About", "Address", \
		"Affiliation", "Author", "Brand", "Citation", \
		"Description", "Email", "Headline", "Keywords", "Language", \
		"Name", "Telephone", "Version"]}]' # to set blog fields
	dms3 index config --json Metadata.Kind [{}] # to reset fields

Use the make document command to make an empty document template
with all the fields pre-generated.

	dms3 index mkdoc -k=blog > b.xml    # edit document
	dms3 index addoc b.xml <path>       # add blog to reposet

The first form of the command (without path argument) is used to
create an infostore. The second form of the command that includes
a path argument is used to create an metastore. A metastore
repository stores meta information for documents contained in an
associated infostore pointed to by the path argument.

`,
	},

	Arguments: []cmds.Argument{
		cmds.StringArg("infostores", false, true, "dms3 path to associated repository.").EnableStdin(),
	},
	Options: []cmds.Option{
		cmds.StringOption(kindOptionName, "k", "keyword for kind of content, ex: \"blog\" ."),
		cmds.StringOption(nameOptionName, "n", "reposet name, ex: \"foodblog\" ."),
		cmds.BoolOption(quietOptionName, "q", "Write just hashes of created object.").WithDefault(false),
		cmds.BoolOption(progressOptionName, "p", "Stream progress data.").WithDefault(true),
	},
	Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error {
		api, err := cmdenv.GetApi(env, req)
		if err != nil {
			return err
		}

		n, err := cmdenv.GetNode(env)
		if err != nil {
			return err
		}

		kopt, _ := req.Options[kindOptionName].(string)
		if kopt == "" {
			return fmt.Errorf("kind of content key must be specified")
		}
		log.Debugf("kind option value %s", kopt)

		nopt, _ := req.Options[nameOptionName].(string)
		if nopt == "" {
			return fmt.Errorf("reposet name must be specified")
		}
		log.Debugf("reposet name option value %s", nopt)

		if len(req.Arguments) < 1 {
			req.SetOption(infoClassName, "infostore")
		} else {
			req.SetOption(infoClassName, "metastore")

		}
		log.Debugf("infoclass is %s", req.Options[infoClassName].(string))

		icfg, err := n.Repo.IdxConfig()
		if err != nil {
			return fmt.Errorf("could not load index config")
		}

		found, err := idxlfs.IsKindConfigured(icfg, kopt)
		if err != nil {
			return err
		}
		if !found {
			return fmt.Errorf("metadata not configured for repo kind %s", kopt)
		}

		// check repo does not already exists on local filesystem
		var rspath, rpath string
		var repoCount int

		if _, rspath, err = idxlfs.ReposetExists(kopt, nopt); err != nil {
			return err
		}

		repoCount = idxlfs.GetRepoCount(rspath)

		rpath = idxlfs.NextRepoPath(rspath, repoCount)
		if rpath != "" {
			return fmt.Errorf("Error repo exists, try using expand option")
		}

		// check repo does not already exists on in kvstore
		var key ds.Key

		iopt, _ := req.Options[infoClassName].(string)

		idxkvs.InitIndexKVStore(n.Repo.Datastore())
		dstore := idxkvs.GetIndexKVStore()

		if key, err = idxkvs.GetRepoSetKey(iopt, kopt, nopt); err != nil {
			return err
		}
		if _, err = dstore.Get(key); err == nil {
			return fmt.Errorf("named reposet key already exists %s", key)
		}
		log.Debugf("reposet key is %v\n", key)

		// create the params file on local filesystem
		var paramsfile, reponame string

		if repoCount == 0 {
			if paramsfile, err = idxlfs.MakeParams(icfg, rspath, kopt); err != nil {
				return err
			}
			log.Debugf("created params file %s", paramsfile)
			fmt.Printf("created params file %s\n", paramsfile)
		} else {
			if paramsfile, err = idxlfs.ParamsLocalFile(kopt); err != nil {
				return err
			}
		}

		_, reponame, ct, err := idxlfs.MakeRepo(icfg, rspath, kopt)
		if err != nil {
			return err
		}
		req.SetOption(createdAtName, fmt.Sprintf("%s", time.Unix(ct, 0).UTC().Format(time.RFC3339)))
		log.Debugf("repo create time %s", req.Options[createdAtName].(string))
		fmt.Printf("repo create time %s\n", req.Options[createdAtName].(string))
		log.Debugf("created repo name %s", reponame)
		fmt.Printf("created repo name %s\n", reponame)

		if err := addParamsFile(req, res, env, api, n, paramsfile, reponame); err != nil {
			return err
		}
		return nil
	},
	PostRun: cmds.PostRunMap{
		cmds.CLI: func(res cmds.Response, re cmds.ResponseEmitter) error {
			sizeChan := make(chan int64, 1)
			outChan := make(chan interface{})
			req := res.Request()

			go func() {
				size, err := req.Files.Size()
				if err != nil {
					log.Warnf("error getting files size: %s", err)

					return
				}

				sizeChan <- size
			}()

			progressBar := func(wait chan struct{}) {
				defer close(wait)

				quiet, _ := req.Options[quietOptionName].(bool)

				quieter := false
				quiet = quiet || quieter

				progress, _ := req.Options[progressOptionName].(bool)

				var bar *pb.ProgressBar
				if progress {
					bar = pb.New64(0).SetUnits(pb.U_BYTES)
					bar.ManualUpdate = true
					bar.ShowTimeLeft = false
					bar.ShowPercent = false
					bar.Output = os.Stderr
					bar.Start()
				}

				lastFile := ""
				lastHash := ""
				var totalProgress, prevFiles, lastBytes int64

			LOOP:
				for {
					select {
					case out, ok := <-outChan:
						if !ok {
							if quieter {
								fmt.Fprintln(os.Stdout, lastHash)
							}

							break LOOP
						}
						output := out.(*IndexMakeEvent)
						if len(output.Hash) > 0 {
							lastHash = output.Hash
							if quieter {
								continue
							}

							if progress {

								fmt.Fprintf(os.Stderr, "\033[2K\r")
							}
							if quiet {
								fmt.Fprintf(os.Stdout, "%s\n", output.Hash)
							} else {
								fmt.Fprintf(os.Stdout, "added %s %s\n", output.Hash, cmdenv.EscNonPrint(output.Name))
							}

						} else {
							if !progress {
								continue
							}

							if len(lastFile) == 0 {
								lastFile = output.Name
							}
							if output.Name != lastFile || output.Bytes < lastBytes {
								prevFiles += lastBytes
								lastFile = output.Name
							}
							lastBytes = output.Bytes
							delta := prevFiles + lastBytes - totalProgress
							totalProgress = bar.Add64(delta)
						}

						if progress {
							bar.Update()
						}
					case size := <-sizeChan:
						if progress {
							bar.Total = size
							bar.ShowPercent = true
							bar.ShowBar = true
							bar.ShowTimeLeft = true
						}
					case <-req.Context.Done():

						return
					}
				}

				if progress && bar.Total == 0 && bar.Get() != 0 {
					bar.Total = bar.Get()
					bar.ShowPercent = true
					bar.ShowBar = true
					bar.ShowTimeLeft = true
					bar.Update()
				}
			}

			if e := res.Error(); e != nil {
				close(outChan)
				return e
			}

			wait := make(chan struct{})
			go progressBar(wait)

			defer func() { <-wait }()
			defer close(outChan)

			for {
				v, err := res.Next()
				if err != nil {
					if err == io.EOF {
						return nil
					}

					return err
				}

				select {
				case outChan <- v:
				case <-req.Context.Done():
					return req.Context.Err()
				}
			}
		},
	},
	Type: IndexMakeEvent{},
}

MakeIndexCmd make a new reposet

var MakeReposetCmd = &cmds.Command{
	Helptext: cmds.HelpText{
		Tagline: "Make index repository set.",
		ShortDescription: `
 Make a new searchable repository set.
 `,
		LongDescription: `
 Make a new searchable infostore or metastore for documents of
 a similar kind. The repository kind is named using a locally
 unique key ex: blog.

 Each created repository set can be customized with specific schema
 fields to expose structure of documents it will host. The exposed
 structure can be used to refine search with the robust supported
 query language.

 The set of fields used for a specifc kind key can be customized
 using the repository configure command.

 	dms3 index config show    # to show index configuration
 	dms3 index config --json Metadata.Kind \
 		'[{"Name": "blog", "Field": ["About", "Address", \
 		"Affiliation", "Author", "Brand", "Citation", \
 		"Description", "Email", "Headline", "Keywords", "Language", \
 		"Name", "Telephone", "Version"]}]' # to set blog fields
 	dms3 index config --json Metadata.Kind [{}] # to reset fields

 Use the make document command to make an empty document template
 with all the fields pre-generated.

 	dms3 index mkdoc -k=blog > b.xml    # edit document
 	dms3 index addoc b.xml <path>       # add blog to reposet

 The meta boolean option controls whether infostore or metastore is created.
 A metastore stores meta information for documents in associated infostores.

 `,
	},

	Arguments: []cmds.Argument{},
	Options: []cmds.Option{
		cmds.StringOption(kindOptionName, "k", "keyword for kind of content, ex: \"blog\" ."),
		cmds.StringOption(nameOptionName, "n", "reposet name, ex: \"foodblog\" ."),
		cmds.BoolOption(quietOptionName, "q", "Write just hashes of created object.").WithDefault(false),
		cmds.BoolOption(metaOptionName, "m", "Create metastore instead of infostore.").WithDefault(false),
		cmds.BoolOption(expandOptionName, "e", "Expand reposet by create a new index repository.").WithDefault(false),
	},
	Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error {

		n, err := cmdenv.GetNode(env)
		if err != nil {
			return err
		}

		kopt, _ := req.Options[kindOptionName].(string)
		if kopt == "" {
			return fmt.Errorf("kind of content key must be specified.")
		} else {
			log.Debugf("requested kind is %s", kopt)
		}

		nopt, _ := req.Options[nameOptionName].(string)
		if nopt == "" {
			return fmt.Errorf("a unique reposet name must be specified.")
		} else {
			log.Debugf("requested reposet name is %s", nopt)
		}

		eopt, _ := req.Options[expandOptionName].(bool)

		icfg, err := n.Repo.IdxConfig()
		if err != nil {
			return fmt.Errorf("could not load index config.")
		}

		if found, err := idxlfs.IsKindConfigured(icfg, kopt); err != nil {
			return err
		} else {
			if !found {
				return fmt.Errorf("metadata not configured for repo kind %s", kopt)
			}
		}

		// metastore?
		var storename, prefix string
		mopt, _ := req.Options[metaOptionName].(bool)
		if mopt {
			prefix = "metastore-"
		} else {
			prefix = "infostore-"
		}
		storename = prefix + nopt

		var rspath, rpath string
		var repoCount int

		if _, rspath, err = idxlfs.ReposetExists(kopt, storename); err != nil {
			return err
		}

		repoCount = idxlfs.GetRepoCount(rspath)

		rpath = idxlfs.NextRepoPath(rspath, repoCount)

		if eopt {
			if rpath == "" {
				return fmt.Errorf("Error reposet size is at limit, repo count is %v", repoCount)
			}
		} else {
			if rpath != "" {
				return fmt.Errorf("Error repo exists, try using expand option")
			}
		}

		// we are creating a repo in reposet
		var paramsfile string

		if repoCount == 0 {
			if paramsfile, err = idxlfs.MakeParams(icfg, rspath, kopt); err != nil {
				return err
			}
			log.Debugf("created params file %s", paramsfile)

		} else {
			if paramsfile, err = idxlfs.ParamsLocalFile(kopt); err != nil {
				return err
			}
		}

		// create repo
		var corpusPath, reponame string
		var ct int64
		if corpusPath, reponame, ct, err = idxlfs.MakeRepo(icfg, rpath, kopt); err != nil {
			return err
		} else {
			req.SetOption(createdAtName, fmt.Sprintf("%s", time.Unix(ct, 0).UTC().Format(time.RFC3339)))
			log.Debugf("repo create time %s", req.Options[createdAtName].(string))

			log.Debugf("created repo name %s", reponame)

		}

		output := RepoInfo{
			Kind:       kopt,
			Name:       nopt,
			StoreName:  storename,
			ParamsPath: paramsfile,
			RepoPath:   rpath,
			CorpusPath: corpusPath,
			IndexRoot:  filepath.Join(rpath, reponame),
			Result:     fmt.Sprintf("Created at %s", req.Options[createdAtName].(string)),
		}

		log.Debugf("output %s", output)

		return cmds.EmitOnce(res, output)
	},
	PostRun: cmds.PostRunMap{
		cmds.CLI: func(res cmds.Response, re cmds.ResponseEmitter) error {
			req := res.Request()

			go func() {
				defer re.Close()
				defer fmt.Println()

				for {
					v, err := res.Next()
					if err == io.EOF {
						return
					}

					if err != nil {
						fmt.Println("\nerror:", err)
						return
					}

					quiet, _ := req.Options[quietOptionName].(bool)
					if quiet {
						s := v.(RepoInfo)
						fmt.Printf("\r%v\n", s.Result)
					} else {
						s := v.(RepoInfo)
						fmt.Printf("\rKind: %v\n", s.Kind)
						fmt.Printf("\rName: %v\n", s.Name)
						fmt.Printf("\rStore name: %v\n", s.StoreName)
						fmt.Printf("\rParams: %v\n", s.ParamsPath)
						fmt.Printf("\rRepo path: %v\n", s.RepoPath)
						fmt.Printf("\rCorpus path: %v\n", s.CorpusPath)
						fmt.Printf("\rIndexRoot root: %v\n", s.IndexRoot)
						fmt.Printf("\r%v\n", s.Result)
					}

				}
			}()

			return nil
		},
	},
	Type: RepoInfo{},
}

var NotyetIndexCmd = &cmds.Command{
	Helptext: cmds.HelpText{
		Tagline: "Not yet implemented.",
		ShortDescription: `
Not yet implemented.
`,
		LongDescription: `
Not yet implemented.
`,
	},
	Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error {

		defer res.Close()

		return res.Emit(fmt.Sprintf("repos: %s", "not implemented yet..."))
	},
	Type: ReposetRefList{},
	PostRun: cmds.PostRunMap{
		cmds.CLI: func(res cmds.Response, re cmds.ResponseEmitter) error {
			if res.Length() > 0 && res.Length() < progressBarMinSize {
				return cmds.Copy(re, res)
			}

			for {
				v, err := res.Next()
				if err != nil {
					if err == io.EOF {
						return nil
					}
					return err
				}
				switch val := v.(type) {
				case io.Reader:
					bar, reader := progressBarForReader(os.Stderr, val, int64(res.Length()))
					bar.Start()

					err = re.Emit(reader)
					if err != nil {
						return err
					}
				default:
					log.Warnf("index ls postrun: received unexpected type %T", val)
				}
			}
		},
	},
}

NotyetIndexCmd command not yet implemented

var PublishIndexCmd = &cmds.Command{
	Helptext: cmds.HelpText{
		Tagline: "Publish index repository.",
		ShortDescription: `
Publish index repository specified by path.
`,
		LongDescription: `
Publish index repository specified by path.
`,
	},

	Arguments: []cmds.Argument{
		cmds.StringArg("dms3-path", true, false, "repository to publish."),
	},
	Options: []cmds.Option{
		cmds.BoolOption("quiet", "q", "Write just hashes of created object."),
	},
	Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error {
		var repo string
		if len(req.Arguments) != 1 {
			return fmt.Errorf("path is required")
		}
		repo = req.Arguments[0]

		log.Debugf("repo path is %s", repo)

		n, err := cmdenv.GetNode(env)
		if err != nil {
			return err
		}

		log.Debugf("Running command request path %s", req.Path)

		pubopts := new(publishOpts)

		q, _ := req.Options["quiet"].(bool)
		if q {
			pubopts.q = true
		} else {
			pubopts.q = false
		}
		log.Debugf("quiet option value %t", q)

		ctx := req.Context

		/*
			p, err := path.ParsePath(p)
			if err != nil {
				return err
			}
		*/
		var p path.Path

		output, err := pubRepo(ctx, n, p, pubopts)
		if err != nil {
			return err
		}
		log.Debugf("output %s", output)

		return cmds.EmitOnce(res, output)
	},
	Encoders: cmds.EncoderMap{
		cmds.Text: cmds.MakeEncoder(func(req *cmds.Request, w io.Writer, v interface{}) error {
			repoPath, ok := v.(*RepoPath)
			if !ok {
				return e.TypeErr(repoPath, v)
			}

			_, err := fmt.Fprintf(w, "path %s \n", repoPath.path)
			return err
		}),
	},
	Type: RepoPath{},
}

PublishIndexCmd publish index repository

var RemoveDocumentCmd = &cmds.Command{
	Helptext: cmds.HelpText{
		Tagline: "Remove document from index repository.",
		ShortDescription: `
Remove document from an index repository specified by path.
`,
		LongDescription: `
Remove document specified by cid from index repository specified by path.
`,
	},

	Arguments: []cmds.Argument{
		cmds.StringArg("cid", true, false, "content to remove from repository."),
		cmds.StringArg("dms3-path", true, false, "repository to remove from."),
	},
	Options: []cmds.Option{
		cmds.BoolOption("quiet", "q", "Write just hashes of created object."),
	},
	Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error {
		var cid, repo string
		if len(req.Arguments) != 2 {
			return fmt.Errorf("cid and path are both required")
		}
		cid = req.Arguments[0]
		repo = req.Arguments[1]

		log.Debugf("cid value is %s, repo path is %s", cid, repo)

		n, err := cmdenv.GetNode(env)
		if err != nil {
			return err
		}

		log.Debugf("Running command request path %s", req.Path)

		rmopts := new(rmdocOpts)

		q, _ := req.Options["quiet"].(bool)
		if q {
			rmopts.q = true
		} else {
			rmopts.q = false
		}
		log.Debugf("quiet option value %t", q)

		ctx := req.Context

		/*
			p, err := path.ParsePath(p)
			if err != nil {
				return err
			}
		*/
		var p path.Path

		output, err := rmDoc(ctx, n, p, rmopts)
		if err != nil {
			return err
		}
		log.Debugf("output %s", output)

		return cmds.EmitOnce(res, output)
	},
	Encoders: cmds.EncoderMap{
		cmds.Text: cmds.MakeEncoder(func(req *cmds.Request, w io.Writer, v interface{}) error {
			repoPath, ok := v.(*RepoPath)
			if !ok {
				return e.TypeErr(repoPath, v)
			}

			_, err := fmt.Fprintf(w, "path %s \n", repoPath.path)
			return err
		}),
	},
	Type: RepoPath{},
}

Types

type IndexMakeEvent

type IndexMakeEvent struct {
	Name  string
	Hash  string `json:",omitempty"`
	Bytes int64  `json:",omitempty"`
	Size  string `json:",omitempty"`
}

type RepoDoc

type RepoDoc struct {
	// contains filtered or unexported fields
}

////////////////////////////////////////////////////////////////////////////

make an empty document for new content

////////////////////////////////////////////////////////////////////////////

type RepoInfo

type RepoInfo struct {
	Kind       string
	Name       string
	StoreName  string
	ParamsPath string
	RepoPath   string
	CorpusPath string
	IndexRoot  string
	Result     string
}

* the prototype implementation of this command above needs to change. * * - want to separate private / public repository management logic. * so when creating a repository, only handle search engine needs. * Specifically, * - not concerned with index recovery (beyond what indri can recover) * - not concerned with p2p sharing/access (defer to publish command) * - want to keep functional code above, add new cmd below with needed logic. * as we can leverage the above working code later when we implement "publish" * * - new command structure: * dms3 index mkidx [--kind=<kind> | -k] [--name=<name> | -n] [--quiet | -q] [--meta | -m] [--expand | -e] * changes: * - remove path argument, use bool option to distinguish meta/infostore * - account of store# within set in the path: * ~/.dms3/index/reposet/<kind>/<name>/<num>/<w-a-c-o> * where, * mkidx => make index reposet * <kind> - kind of store * <name> - name of store, metaspace- or infospace- prefix is added * - metastore / infostore association is by common <kind> key * - additional infostores are associated with existing metastore * - publish shares repository with network peers via the permanent file system * - roadmap * validate params / options * make sure kind is properly configured * make sure repo does not exist (meta or info), unless --expand is true * compute # of repos within set, make sure repo num >0, <256 * if repo num == 1, create params file and sub-directories * create index repo, using infospace go interface *

type RepoPath

type RepoPath struct {
	// contains filtered or unexported fields
}

RepoPath path to index repository

type ReposetRef

type ReposetRef struct {
	Infoclass   string
	Reposetkind string
	Reposetname string
	Reposetpath string
}

ReposetRef provides reposet properties

type ReposetRefList

type ReposetRefList []ReposetRef

ReposetRefList provides a list of reposet properties