Skip to content
Snippets Groups Projects
Commit dc2e343a authored by Jeromy Johnson's avatar Jeromy Johnson
Browse files

add option to disable flushing files structure on writes


License: MIT
Signed-off-by: default avatarJeromy <jeromyj@gmail.com>
parent 89ba9942
No related branches found
No related tags found
1 merge request!1New
......@@ -68,7 +68,7 @@ var FilesStatCmd = &cmds.Command{
return
}
o, err := statNode(fsn)
o, err := statNode(node.DAG, fsn)
if err != nil {
res.SetError(err, cmds.ErrNormal)
return
......@@ -90,13 +90,14 @@ var FilesStatCmd = &cmds.Command{
Type: Object{},
}
func statNode(fsn mfs.FSNode) (*Object, error) {
func statNode(ds dag.DAGService, fsn mfs.FSNode) (*Object, error) {
nd, err := fsn.GetNode()
if err != nil {
return nil, err
}
k, err := nd.Key()
// add to dagserv to ensure its available
k, err := ds.Add(nd)
if err != nil {
return nil, err
}
......@@ -434,10 +435,20 @@ a beginning offset to write to. The entire length of the input will be written.
If the '--create' option is specified, the file will be created if it does not
exist. Nonexistant intermediate directories will not be created.
If the '--flush' option is set to false, changes will not be propogated to the
merkledag root. This can make operations much faster when doing a large number
of writes to a deeper directory structure.
Example:
echo "hello world" | ipfs files write --create /myfs/a/b/file
echo "hello world" | ipfs files write --truncate /myfs/a/b/file
Warning:
Usage of the '--flush=false' option does not guarantee data durability until
the tree has been flushed. This can be accomplished by running 'ipfs files stat'
on the file or any of its ancestors.
`,
},
Arguments: []cmds.Argument{
......@@ -449,6 +460,7 @@ Example:
cmds.BoolOption("e", "create", "create the file if it does not exist"),
cmds.BoolOption("t", "truncate", "truncate the file before writing"),
cmds.IntOption("n", "count", "maximum number of bytes to read"),
cmds.BoolOption("f", "flush", "flush file and ancestors after write (default: true)"),
},
Run: func(req cmds.Request, res cmds.Response) {
path, err := checkPath(req.Arguments()[0])
......@@ -459,6 +471,10 @@ Example:
create, _, _ := req.Option("create").Bool()
trunc, _, _ := req.Option("truncate").Bool()
flush, set, _ := req.Option("flush").Bool()
if !set {
flush = true
}
nd, err := req.InvocContext().GetNode()
if err != nil {
......@@ -471,7 +487,12 @@ Example:
res.SetError(err, cmds.ErrNormal)
return
}
if flush {
defer fi.Close()
} else {
defer fi.Sync()
}
if trunc {
if err := fi.Truncate(0); err != nil {
......
......@@ -53,7 +53,16 @@ func (d *Directory) closeChild(name string, nd *dag.Node) error {
d.lock.Lock()
defer d.lock.Unlock()
err = d.node.RemoveNodeLink(name)
err = d.updateChild(name, nd)
if err != nil {
return err
}
return d.parent.closeChild(d.name, d.node)
}
func (d *Directory) updateChild(name string, nd *dag.Node) error {
err := d.node.RemoveNodeLink(name)
if err != nil && err != dag.ErrNotFound {
return err
}
......@@ -63,7 +72,7 @@ func (d *Directory) closeChild(name string, nd *dag.Node) error {
return err
}
return d.parent.closeChild(d.name, d.node)
return nil
}
func (d *Directory) Type() NodeType {
......@@ -77,30 +86,16 @@ func (d *Directory) childFile(name string) (*File, error) {
return fi, nil
}
nd, err := d.childFromDag(name)
if err != nil {
return nil, err
}
i, err := ft.FromBytes(nd.Data)
fsn, err := d.childNode(name)
if err != nil {
return nil, err
}
switch i.GetType() {
case ufspb.Data_Directory:
return nil, ErrIsDirectory
case ufspb.Data_File:
nfi, err := NewFile(name, nd, d, d.dserv)
if err != nil {
return nil, err
}
d.files[name] = nfi
return nfi, nil
case ufspb.Data_Metadata:
return nil, ErrNotYetImplemented
default:
return nil, ErrInvalidChild
if fi, ok := fsn.(*File); ok {
return fi, nil
}
return nil, fmt.Errorf("%s is not a file", name)
}
// childDir returns a directory under this directory by the given name if it
......@@ -111,6 +106,21 @@ func (d *Directory) childDir(name string) (*Directory, error) {
return dir, nil
}
fsn, err := d.childNode(name)
if err != nil {
return nil, err
}
if dir, ok := fsn.(*Directory); ok {
return dir, nil
}
return nil, fmt.Errorf("%s is not a directory", name)
}
// childNode returns a FSNode under this directory by the given name if it exists.
// it does *not* check the cached dirs and files
func (d *Directory) childNode(name string) (FSNode, error) {
nd, err := d.childFromDag(name)
if err != nil {
return nil, err
......@@ -127,7 +137,12 @@ func (d *Directory) childDir(name string) (*Directory, error) {
d.childDirs[name] = ndir
return ndir, nil
case ufspb.Data_File:
return nil, fmt.Errorf("%s is not a directory", name)
nfi, err := NewFile(name, nd, d, d.dserv)
if err != nil {
return nil, err
}
d.files[name] = nfi
return nfi, nil
case ufspb.Data_Metadata:
return nil, ErrNotYetImplemented
default:
......@@ -157,17 +172,17 @@ func (d *Directory) Child(name string) (FSNode, error) {
// childUnsync returns the child under this directory by the given name
// without locking, useful for operations which already hold a lock
func (d *Directory) childUnsync(name string) (FSNode, error) {
dir, err := d.childDir(name)
if err == nil {
return dir, nil
cdir, ok := d.childDirs[name]
if ok {
return cdir, nil
}
fi, err := d.childFile(name)
if err == nil {
return fi, nil
cfile, ok := d.files[name]
if ok {
return cfile, nil
}
return nil, os.ErrNotExist
return d.childNode(name)
}
type NodeListing struct {
......@@ -305,7 +320,53 @@ func (d *Directory) AddChild(name string, nd *dag.Node) error {
return d.parent.closeChild(d.name, d.node)
}
func (d *Directory) sync() error {
for name, dir := range d.childDirs {
nd, err := dir.GetNode()
if err != nil {
return err
}
_, err = d.dserv.Add(nd)
if err != nil {
return err
}
err = d.updateChild(name, nd)
if err != nil {
return err
}
}
for name, file := range d.files {
nd, err := file.GetNode()
if err != nil {
return err
}
_, err = d.dserv.Add(nd)
if err != nil {
return err
}
err = d.updateChild(name, nd)
if err != nil {
return err
}
}
return nil
}
func (d *Directory) GetNode() (*dag.Node, error) {
d.Lock()
defer d.Unlock()
err := d.sync()
if err != nil {
return nil, err
}
return d.node, nil
}
......
......@@ -316,13 +316,26 @@ test_files_api() {
verify_dir_contents /cats file1 ipfs this
'
test_expect_success "write 'no-flush' succeeds" '
echo "testing" | ipfs files write -f -e /cats/walrus
'
test_expect_success "changes bubbled up to root on inspection" '
ipfs files stat / | head -n1 > root_hash
'
test_expect_success "root hash looks good" '
echo "QmcwKfTMCT7AaeiD92hWjnZn9b6eh9NxnhfSzN5x2vnDpt" > root_hash_exp &&
test_cmp root_hash_exp root_hash
'
# test mv
test_expect_success "can mv dir" '
ipfs files mv /cats/this/is /cats/
'
test_expect_success "mv worked" '
verify_dir_contents /cats file1 ipfs this is &&
verify_dir_contents /cats file1 ipfs this is walrus &&
verify_dir_contents /cats/this
'
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment