// // decsync-vdir // Copyright © 2022 by luk3yx // // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU Affero General Public License as // published by the Free Software Foundation, either version 3 of the // License, or (at your option) any later version. // // This program is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU Affero General Public License for more details. // // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . // package main import ( "bufio" "bytes" cryptoRand "crypto/rand" "encoding/json" "errors" "fmt" "io" "os" "regexp" "strings" "time" ) type any = interface{} var infoPath = []string{"info"} var ErrNoDeviceName = errors.New("No device name specified (read-only mode)") var safeFilenameRegex = regexp.MustCompile(`^[A-Za-z0-9\-_]+$`) var storageFileRegex = regexp.MustCompile(`^[0-9A-Fa-f]{2}|info$`) func isUnsafeFilename(fn string) bool { return safeFilenameRegex.Find([]byte(fn)) == nil } func assert(err error) { if err != nil { panic(err) } } func uuid4() string { var uuid [16]byte _, err := cryptoRand.Read(uuid[:]) assert(err) uuid[6] = (uuid[6] & 0x0f) | 0x40 uuid[8] = (uuid[8] & 0x3f) | 0x80 s := fmt.Sprintf("%032x", uuid) return fmt.Sprintf("%s-%s-%s-%s-%s", s[:8], s[8:12], s[12:16], s[16:20], s[20:]) } // This code doesn't work exactly like libdecsync, it doesn't bother updating // its own entry files with any new entries. func getPathHash(path []string) string { if len(path) == 1 && path[0] == "info" { return "info" } var hash uint8 for _, component := range path { var componentHash uint8 for _, b := range []byte(component) { componentHash = componentHash*19 + b } hash = hash*199 + componentHash } return fmt.Sprintf("%02x", hash) } func pathEquals(p1, p2 []string) bool { if len(p1) != len(p2) { return false } for i, v1 := range p1 { if v1 != p2[i] { return false } } return true } func pathStartsWith(p1, p2 []string) bool { if len(p1) < len(p2) { return false } for i, v2 := range p2 { if v2 != p1[i] { return false } } return true } func isDir(path string) (bool, error) { info, err := os.Stat(path) if errors.Is(err, os.ErrNotExist) { return false, nil } else if err != nil { return false, err } return info.IsDir(), nil } type DecSyncFolder struct { directory string deviceName string lastActiveDay time.Time } type SyncEntry struct { deviceName string Path []string LastModified time.Time Key any Value any } const decsyncTimeFormat = "2006-01-02T15:04:05" func (e *SyncEntry) UnmarshalJSON(data []byte) error { var dateStr string var arr = [4]any{&e.Path, &dateStr, &e.Key, &e.Value} err := json.Unmarshal(data, &arr) if err == nil { e.LastModified, err = time.Parse(decsyncTimeFormat, dateStr) } return err } func (e SyncEntry) MarshalJSON() ([]byte, error) { dateStr := e.LastModified.UTC().Format(decsyncTimeFormat) return json.Marshal([4]any{e.Path, dateStr, e.Key, e.Value}) } func (d *DecSyncFolder) Exists() (bool, error) { return isDir(d.directory + "/v2") } func (d *DecSyncFolder) readAllEntries(path []string, callback func(SyncEntry)) error { deviceDirs, err := os.ReadDir(d.directory + "/v2") if err != nil { return err } hash := getPathHash(path) for _, dir := range deviceDirs { err = d.parseFile(dir.Name(), hash, callback) if err != nil && !errors.Is(err, os.ErrNotExist) { return err } } return nil } func (d *DecSyncFolder) parseFile(deviceName, hash string, callback func(SyncEntry)) error { f, err := os.Open(d.directory + "/v2/" + deviceName + "/" + hash) if err != nil { return err } defer f.Close() scanner := bufio.NewScanner(f) for scanner.Scan() { var entry SyncEntry if err = json.Unmarshal(scanner.Bytes(), &entry); err != nil { return err } entry.deviceName = deviceName callback(entry) } return nil } func insertEntryIfNewer(entries map[any]SyncEntry, entry SyncEntry) { curEntry := entries[entry.Key] if entry.LastModified.After(curEntry.LastModified) { entries[entry.Key] = entry } } // Iterates over everything starting with the path prefix func (d *DecSyncFolder) Iter(pathPrefix []string, includeDeleted bool, callback func(SyncEntry) error) error { deviceDirs, err := os.ReadDir(d.directory + "/v2") if err != nil { return err } // Collect all the filenames that need to be searched hashes := make(map[string][]string) for _, dir := range deviceDirs { // The remote device's "sequences" file may not list all the files, // as data that is imported from a now-deleted device doesn't make // DecSync increment the sequence numbers. files, err := os.ReadDir(d.directory + "/v2/" + dir.Name()) if err != nil { return err } for _, fn := range files { hash := fn.Name() if storageFileRegex.Find([]byte(hash)) != nil { hashes[hash] = append(hashes[hash], dir.Name()) } } } // Now read all entries one file at a time for hash, devicesWithHash := range hashes { // Create an entries map from all devices that have the file entries := make(map[any]SyncEntry) for _, deviceName := range devicesWithHash { err = d.parseFile(deviceName, hash, func(entry SyncEntry) { if pathStartsWith(entry.Path, pathPrefix) { insertEntryIfNewer(entries, entry) } }) if err != nil { return err } } // Run the callbacks on this "batch" of entries for _, entry := range entries { // Don't process deleted entries if !includeDeleted && entry.Value == nil { continue } if err = callback(entry); err != nil { return err } } } return nil } // Gets all values in a path func (d *DecSyncFolder) GetAll(path []string) (map[any]any, error) { entries := make(map[any]SyncEntry) err := d.readAllEntries(path, func(entry SyncEntry) { if pathEquals(entry.Path, path) { insertEntryIfNewer(entries, entry) } }) if err != nil { return nil, err } values := make(map[any]any, len(entries)) for k, entry := range entries { if entry.Value != nil { values[k] = entry.Value } } return values, nil } // Gets one value from a path func (d *DecSyncFolder) Get(path []string, key any) (any, error) { var curEntry SyncEntry err := d.readAllEntries(path, func(entry SyncEntry) { if pathEquals(entry.Path, path) && key == entry.Key && entry.LastModified.After(curEntry.LastModified) { curEntry = entry } }) return curEntry.Value, err } // Gets the name (or an empty string on error) func (d *DecSyncFolder) Name() string { name, err := d.Get(infoPath, "name") if err != nil { return "" } nameStr, _ := name.(string) return nameStr } func writeIfNotExist(dir, fn string, content []byte) error { if err := os.MkdirAll(dir, 0750); err != nil { return err } f, err := os.OpenFile(dir+"/"+fn, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0640) if err != nil { if errors.Is(err, os.ErrExist) { return nil } return err } defer f.Close() _, err = f.Write(content) return err } func (d *DecSyncFolder) writeDeviceInfo() error { if d.deviceName == "" { return ErrNoDeviceName } // Create the "local" directory localDeviceDir := d.directory + "/local/" + d.deviceName if err := os.MkdirAll(localDeviceDir, 0750); err != nil { return err } // If the last-active value was already updated today then don't bother // updating it again utcNow := time.Now().UTC() utcToday := utcNow.Truncate(24 * time.Hour) if d.lastActiveDay.Equal(utcToday) { return nil } // Write to the "info" file lastActiveStr := utcToday.Format("2006-01-02") data, err := json.Marshal(map[string]any{ "version": 2, "last-active": lastActiveStr, "supported-version": 2, }) assert(err) data = append(data, '\n') if err = os.WriteFile(localDeviceDir+"/info", data, 0640); err != nil { return err } // Make sure our /v2/ directory exists if err = os.MkdirAll(d.directory+"/v2/"+d.deviceName, 0750); err != nil { return err } // Update the "last active" and "supported version" entries err = d.insertEntryRaw(SyncEntry{ Path: infoPath, LastModified: utcNow, Key: "last-active-" + d.deviceName, Value: lastActiveStr, }) if err == nil { err = d.insertEntryRaw(SyncEntry{ Path: infoPath, LastModified: utcNow, Key: "supported-version-" + d.deviceName, Value: 2, }) if err == nil { d.lastActiveDay = utcToday } } return err } func atomicWriteFile(fn string, data []byte) error { tmpFn := fn + ".tmp" if err := os.WriteFile(tmpFn, data, 0640); err != nil { return err } return os.Rename(tmpFn, fn) } // Sets a value func (d *DecSyncFolder) Set(path []string, key, value any) error { return d.InsertEntry(SyncEntry{ Path: path, LastModified: time.Now(), Key: key, Value: value, }) } func (d *DecSyncFolder) InsertEntry(newEntry SyncEntry) error { if err := d.writeDeviceInfo(); err != nil { return err } return d.insertEntryRaw(newEntry) } func (d *DecSyncFolder) insertEntryRaw(newEntry SyncEntry) error { // Read existing entries entries := make([]SyncEntry, 0, 1) hash := getPathHash(newEntry.Path) err := d.parseFile(d.deviceName, hash, func(entry SyncEntry) { if !pathEquals(newEntry.Path, entry.Path) || newEntry.Key != entry.Key { entries = append(entries, entry) } }) if err != nil && !errors.Is(err, os.ErrNotExist) { return err } // Read the sequences file v2Dir := d.directory + "/v2/" + d.deviceName + "/" sequencesFn := v2Dir + "sequences" data, err := os.ReadFile(sequencesFn) if err != nil { if !errors.Is(err, os.ErrNotExist) { return err } data = []byte("{}") } // Increment the modified hash var sequences map[string]uint64 if err = json.Unmarshal(data, &sequences); err != nil { return err } sequences[hash]++ // Write the sequences file back // This is done before writing the hash file because it's probably safe to // increment the sequences number without changing anything data, err = json.Marshal(sequences) assert(err) if err = atomicWriteFile(sequencesFn, append(data, '\n')); err != nil { return err } // Add new entry entries = append(entries, newEntry) // Write all the entries back as JSON var b bytes.Buffer for _, entry := range entries { data, err := json.Marshal(entry) if err != nil { return err } b.Write(data) b.WriteByte('\n') } return atomicWriteFile(v2Dir+hash, b.Bytes()) } // For handling contacts/calendars func (d *DecSyncFolder) GetResource(uid string) (string, error) { resource, err := d.Get([]string{"resources", uid}, nil) if err != nil || resource == nil { return "", err } else if resourceStr, ok := resource.(string); ok { return resourceStr, nil } return "", errors.New("Invalid resource type") } func (d *DecSyncFolder) UpdateResource(uid string, newData string, ts time.Time) error { return d.InsertEntry(SyncEntry{ Path: []string{"resources", uid}, LastModified: ts, Key: nil, Value: newData, }) } func (d *DecSyncFolder) DeleteResource(uid string) error { return d.Set([]string{"resources", uid}, nil, nil) } func (d *DecSyncFolder) CreateResource(data string, ts time.Time) (string, error) { uid := uuid4() return uid, d.UpdateResource(uid, data, ts) } func (d *DecSyncFolder) IterResources(includeDeleted bool, callback func(string, string, time.Time) error) error { return d.Iter([]string{"resources"}, includeDeleted, func(entry SyncEntry) error { if entry.Key != nil { return nil } data, ok := entry.Value.(string) if !ok && entry.Value != nil { return errors.New("Invalid resource type") } return callback(entry.Path[1], data, entry.LastModified) }) } func getVcardUID(reader io.Reader) (string, error) { scanner := bufio.NewScanner(reader) for scanner.Scan() { if bytes.HasPrefix(scanner.Bytes(), []byte("UID:")) { return string(scanner.Bytes()[4:]), nil } } return "", scanner.Err() } type uidToPathMap struct { m map[string]string local map[string]struct{} vdir string fileExtension string } func (m uidToPathMap) addFile(vdirFile string) error { f, err := os.Open(vdirFile) if err != nil { return err } defer f.Close() uid, err := getVcardUID(f) if err == nil { if uid == "" { return errors.New("Could not read UID from " + vdirFile) } else if otherFile, exists := m.m[uid]; exists { return fmt.Errorf("Files %q and %q have the same UID", vdirFile, otherFile) } m.m[uid] = vdirFile m.local[uid] = struct{}{} } return err } func (m uidToPathMap) getPath(uid string) string { if path, ok := m.m[uid]; ok { return path } else if isUnsafeFilename(uid) { uid = uuid4() } return m.vdir + uid + m.fileExtension } func SyncVdir(sync *DecSyncFolder, vdir, fileExtension string) error { if !strings.HasSuffix(vdir, "/") { vdir += "/" } if fileExtension != "" && (fileExtension[0] != '.' || isUnsafeFilename(fileExtension[1:])) { return fmt.Errorf("Invalid file extension: %q", fileExtension) } if err := os.MkdirAll(vdir, 0750); err != nil { return err } m := uidToPathMap{ m: make(map[string]string), local: make(map[string]struct{}), vdir: vdir, fileExtension: fileExtension, } files, err := os.ReadDir(vdir) if err != nil { return err } for _, file := range files { if !strings.HasSuffix(file.Name(), fileExtension) { continue } if err = m.addFile(vdir + file.Name()); err != nil { return err } } // Sync contacts that exist on DecSync err = sync.IterResources(true, func(uid, syncData string, syncModified time.Time) error { if isUnsafeFilename(uid) { return errors.New("UUID is not a safe filename") } // The file was "seen" by DecSync so it isn't local delete(m.local, uid) vdirFile := m.getPath(uid) info, err := os.Stat(vdirFile) var vdirModified time.Time if err == nil { vdirModified = info.ModTime().Truncate(time.Second) } else if !errors.Is(err, os.ErrNotExist) { return err } else if syncData == "" { // fmt.Println("Deleted on both sides: " + vdirFile) return nil } if syncModified.Before(vdirModified) { // vdir → DecSync fmt.Println("Updating DecSync: "+vdirFile, syncModified, vdirModified) content, err := os.ReadFile(vdirFile) if err != nil { return err } return sync.UpdateResource(uid, string(content), vdirModified) } else if vdirModified.Before(syncModified) { // DecSync → vdir // fmt.Println("Updating vdir: "+vdirFile, syncModified, vdirModified) if syncData == "" { return os.Remove(vdirFile) } err = atomicWriteFile(vdirFile, []byte(syncData)) if err == nil { err = os.Chtimes(vdirFile, time.Now(), syncModified) } return err } else { // fmt.Println("Not modified: " + vdirFile) return nil } }) for uid := range m.local { vdirFile, ok := m.m[uid] if !ok { panic("Unreachable code") } fmt.Println("Creating on DecSync: " + vdirFile) info, err := os.Stat(vdirFile) if err != nil { return err } vdirModified := info.ModTime().Truncate(time.Second) content, err := os.ReadFile(vdirFile) if err != nil { return err } if err = sync.UpdateResource(uid, string(content), vdirModified); err != nil { return err } } return nil } func main() { if len(os.Args) != 4 { fmt.Fprintln(os.Stderr, "Usage: decsync-vdir /path/to/decsync/contacts/uuid /path/to/vdir .vcf") os.Exit(1) } deviceName, err := os.Hostname() assert(err) if isUnsafeFilename(deviceName) { fmt.Fprintf(os.Stderr, "Warning: This system's hostname (%q) cannot "+ "be used as a filename, read-only mode enabled.\n", deviceName) deviceName = "" } sync := DecSyncFolder{ directory: os.Args[1], deviceName: deviceName, } syncExists, err := sync.Exists() assert(err) if !syncExists { fmt.Fprintln(os.Stderr, "Error: The specified DecSync directory doesn't look valid!") // Try to print a more helpful error message possiblyRootDecsyncDir, err := isDir(sync.directory + "/contacts") assert(err) if !possiblyRootDecsyncDir { possiblyRootDecsyncDir, err = isDir(sync.directory + "/calendars") assert(err) } if possiblyRootDecsyncDir { fmt.Fprintf(os.Stderr, "Try using %[1]s/contacts/ or "+ "%[1]s/calendars/ instead.\n", sync.directory) fmt.Fprintf(os.Stderr, " can be found with "+ "`ls %q/{contacts,calendars}`\n", sync.directory) } else { fmt.Fprintf(os.Stderr, "If know what you're doing, you can run "+ "`mkdir %q` to ignore this error.\n", sync.directory+"/v2") } os.Exit(1) } // fmt.Println("Syncing...") err = SyncVdir(&sync, os.Args[2], os.Args[3]) if err != nil { fmt.Fprintf(os.Stderr, "Error: %s\n", err.Error()) os.Exit(1) } // fmt.Println("Synced!") }