Example File Systems

pyfuse3 comes with several example file systems in the examples directory of the release tarball. For completeness, these examples are also included here.

Single-file, Read-only File System

(shipped as examples/lltest.py)

  1#!/usr/bin/env python3
  2# -*- coding: utf-8 -*-
  3'''
  4hello.py - Example file system for pyfuse3.
  5
  6This program presents a static file system containing a single file.
  7
  8Copyright © 2015 Nikolaus Rath <Nikolaus.org>
  9Copyright © 2015 Gerion Entrup.
 10
 11Permission is hereby granted, free of charge, to any person obtaining a copy of
 12this software and associated documentation files (the "Software"), to deal in
 13the Software without restriction, including without limitation the rights to
 14use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
 15the Software, and to permit persons to whom the Software is furnished to do so.
 16
 17THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 18IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
 19FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
 20COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
 21IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 22CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 23'''
 24
 25import os
 26import sys
 27
 28# If we are running from the pyfuse3 source directory, try
 29# to load the module from there first.
 30basedir = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '..'))
 31if (os.path.exists(os.path.join(basedir, 'setup.py')) and
 32    os.path.exists(os.path.join(basedir, 'src', 'pyfuse3', '__init__.pyx'))):
 33    sys.path.insert(0, os.path.join(basedir, 'src'))
 34
 35from argparse import ArgumentParser
 36import stat
 37import logging
 38import errno
 39import pyfuse3
 40import trio
 41
 42try:
 43    import faulthandler
 44except ImportError:
 45    pass
 46else:
 47    faulthandler.enable()
 48
 49log = logging.getLogger(__name__)
 50
 51class TestFs(pyfuse3.Operations):
 52    def __init__(self):
 53        super(TestFs, self).__init__()
 54        self.hello_name = b"message"
 55        self.hello_inode = pyfuse3.ROOT_INODE+1
 56        self.hello_data = b"hello world\n"
 57
 58    async def getattr(self, inode, ctx=None):
 59        entry = pyfuse3.EntryAttributes()
 60        if inode == pyfuse3.ROOT_INODE:
 61            entry.st_mode = (stat.S_IFDIR | 0o755)
 62            entry.st_size = 0
 63        elif inode == self.hello_inode:
 64            entry.st_mode = (stat.S_IFREG | 0o644)
 65            entry.st_size = len(self.hello_data)
 66        else:
 67            raise pyfuse3.FUSEError(errno.ENOENT)
 68
 69        stamp = int(1438467123.985654 * 1e9)
 70        entry.st_atime_ns = stamp
 71        entry.st_ctime_ns = stamp
 72        entry.st_mtime_ns = stamp
 73        entry.st_gid = os.getgid()
 74        entry.st_uid = os.getuid()
 75        entry.st_ino = inode
 76
 77        return entry
 78
 79    async def lookup(self, parent_inode, name, ctx=None):
 80        if parent_inode != pyfuse3.ROOT_INODE or name != self.hello_name:
 81            raise pyfuse3.FUSEError(errno.ENOENT)
 82        return await self.getattr(self.hello_inode)
 83
 84    async def opendir(self, inode, ctx):
 85        if inode != pyfuse3.ROOT_INODE:
 86            raise pyfuse3.FUSEError(errno.ENOENT)
 87        return inode
 88
 89    async def readdir(self, fh, start_id, token):
 90        assert fh == pyfuse3.ROOT_INODE
 91
 92        # only one entry
 93        if start_id == 0:
 94            pyfuse3.readdir_reply(
 95                token, self.hello_name, await self.getattr(self.hello_inode), 1)
 96        return
 97
 98    async def open(self, inode, flags, ctx):
 99        if inode != self.hello_inode:
100            raise pyfuse3.FUSEError(errno.ENOENT)
101        if flags & os.O_RDWR or flags & os.O_WRONLY:
102            raise pyfuse3.FUSEError(errno.EACCES)
103        return pyfuse3.FileInfo(fh=inode)
104
105    async def read(self, fh, off, size):
106        assert fh == self.hello_inode
107        return self.hello_data[off:off+size]
108
109def init_logging(debug=False):
110    formatter = logging.Formatter('%(asctime)s.%(msecs)03d %(threadName)s: '
111                                  '[%(name)s] %(message)s', datefmt="%Y-%m-%d %H:%M:%S")
112    handler = logging.StreamHandler()
113    handler.setFormatter(formatter)
114    root_logger = logging.getLogger()
115    if debug:
116        handler.setLevel(logging.DEBUG)
117        root_logger.setLevel(logging.DEBUG)
118    else:
119        handler.setLevel(logging.INFO)
120        root_logger.setLevel(logging.INFO)
121    root_logger.addHandler(handler)
122
123def parse_args():
124    '''Parse command line'''
125
126    parser = ArgumentParser()
127
128    parser.add_argument('mountpoint', type=str,
129                        help='Where to mount the file system')
130    parser.add_argument('--debug', action='store_true', default=False,
131                        help='Enable debugging output')
132    parser.add_argument('--debug-fuse', action='store_true', default=False,
133                        help='Enable FUSE debugging output')
134    return parser.parse_args()
135
136
137def main():
138    options = parse_args()
139    init_logging(options.debug)
140
141    testfs = TestFs()
142    fuse_options = set(pyfuse3.default_options)
143    fuse_options.add('fsname=hello')
144    if options.debug_fuse:
145        fuse_options.add('debug')
146    pyfuse3.init(testfs, options.mountpoint, fuse_options)
147    try:
148        trio.run(pyfuse3.main)
149    except:
150        pyfuse3.close(unmount=False)
151        raise
152
153    pyfuse3.close()
154
155
156if __name__ == '__main__':
157    main()

In-memory File System

(shipped as examples/tmpfs.py)

  1#!/usr/bin/env python3
  2# -*- coding: utf-8 -*-
  3'''
  4tmpfs.py - Example file system for pyfuse3.
  5
  6This file system stores all data in memory.
  7
  8Copyright © 2013 Nikolaus Rath <Nikolaus.org>
  9
 10Permission is hereby granted, free of charge, to any person obtaining a copy of
 11this software and associated documentation files (the "Software"), to deal in
 12the Software without restriction, including without limitation the rights to
 13use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
 14the Software, and to permit persons to whom the Software is furnished to do so.
 15
 16THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 17IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
 18FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
 19COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
 20IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 21CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 22'''
 23
 24import os
 25import sys
 26
 27# If we are running from the pyfuse3 source directory, try
 28# to load the module from there first.
 29basedir = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '..'))
 30if (os.path.exists(os.path.join(basedir, 'setup.py')) and
 31    os.path.exists(os.path.join(basedir, 'src', 'pyfuse3', '__init__.pyx'))):
 32    sys.path.insert(0, os.path.join(basedir, 'src'))
 33
 34import pyfuse3
 35import errno
 36import stat
 37from time import time
 38import sqlite3
 39import logging
 40from collections import defaultdict
 41from pyfuse3 import FUSEError
 42from argparse import ArgumentParser
 43import trio
 44
 45try:
 46    import faulthandler
 47except ImportError:
 48    pass
 49else:
 50    faulthandler.enable()
 51
 52log = logging.getLogger()
 53
 54class Operations(pyfuse3.Operations):
 55    '''An example filesystem that stores all data in memory
 56
 57    This is a very simple implementation with terrible performance.
 58    Don't try to store significant amounts of data. Also, there are
 59    some other flaws that have not been fixed to keep the code easier
 60    to understand:
 61
 62    * atime, mtime and ctime are not updated
 63    * generation numbers are not supported
 64    * lookup counts are not maintained
 65    '''
 66
 67    enable_writeback_cache = True
 68
 69    def __init__(self):
 70        super(Operations, self).__init__()
 71        self.db = sqlite3.connect(':memory:')
 72        self.db.text_factory = str
 73        self.db.row_factory = sqlite3.Row
 74        self.cursor = self.db.cursor()
 75        self.inode_open_count = defaultdict(int)
 76        self.init_tables()
 77
 78    def init_tables(self):
 79        '''Initialize file system tables'''
 80
 81        self.cursor.execute("""
 82        CREATE TABLE inodes (
 83            id        INTEGER PRIMARY KEY,
 84            uid       INT NOT NULL,
 85            gid       INT NOT NULL,
 86            mode      INT NOT NULL,
 87            mtime_ns  INT NOT NULL,
 88            atime_ns  INT NOT NULL,
 89            ctime_ns  INT NOT NULL,
 90            target    BLOB(256) ,
 91            size      INT NOT NULL DEFAULT 0,
 92            rdev      INT NOT NULL DEFAULT 0,
 93            data      BLOB
 94        )
 95        """)
 96
 97        self.cursor.execute("""
 98        CREATE TABLE contents (
 99            rowid     INTEGER PRIMARY KEY AUTOINCREMENT,
100            name      BLOB(256) NOT NULL,
101            inode     INT NOT NULL REFERENCES inodes(id),
102            parent_inode INT NOT NULL REFERENCES inodes(id),
103
104            UNIQUE (name, parent_inode)
105        )""")
106
107        # Insert root directory
108        now_ns = int(time() * 1e9)
109        self.cursor.execute("INSERT INTO inodes (id,mode,uid,gid,mtime_ns,atime_ns,ctime_ns) "
110                            "VALUES (?,?,?,?,?,?,?)",
111                            (pyfuse3.ROOT_INODE, stat.S_IFDIR | stat.S_IRUSR | stat.S_IWUSR
112                              | stat.S_IXUSR | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH
113                              | stat.S_IXOTH, os.getuid(), os.getgid(), now_ns, now_ns, now_ns))
114        self.cursor.execute("INSERT INTO contents (name, parent_inode, inode) VALUES (?,?,?)",
115                            (b'..', pyfuse3.ROOT_INODE, pyfuse3.ROOT_INODE))
116
117
118    def get_row(self, *a, **kw):
119        self.cursor.execute(*a, **kw)
120        try:
121            row = next(self.cursor)
122        except StopIteration:
123            raise NoSuchRowError()
124        try:
125            next(self.cursor)
126        except StopIteration:
127            pass
128        else:
129            raise NoUniqueValueError()
130
131        return row
132
133    async def lookup(self, inode_p, name, ctx=None):
134        if name == '.':
135            inode = inode_p
136        elif name == '..':
137            inode = self.get_row("SELECT * FROM contents WHERE inode=?",
138                                 (inode_p,))['parent_inode']
139        else:
140            try:
141                inode = self.get_row("SELECT * FROM contents WHERE name=? AND parent_inode=?",
142                                     (name, inode_p))['inode']
143            except NoSuchRowError:
144                raise(pyfuse3.FUSEError(errno.ENOENT))
145
146        return await self.getattr(inode, ctx)
147
148
149    async def getattr(self, inode, ctx=None):
150        try:
151            row = self.get_row("SELECT * FROM inodes WHERE id=?", (inode,))
152        except NoSuchRowError:
153            raise(pyfuse3.FUSEError(errno.ENOENT))
154
155        entry = pyfuse3.EntryAttributes()
156        entry.st_ino = inode
157        entry.generation = 0
158        entry.entry_timeout = 300
159        entry.attr_timeout = 300
160        entry.st_mode = row['mode']
161        entry.st_nlink = self.get_row("SELECT COUNT(inode) FROM contents WHERE inode=?",
162                                     (inode,))[0]
163        entry.st_uid = row['uid']
164        entry.st_gid = row['gid']
165        entry.st_rdev = row['rdev']
166        entry.st_size = row['size']
167
168        entry.st_blksize = 512
169        entry.st_blocks = 1
170        entry.st_atime_ns = row['atime_ns']
171        entry.st_mtime_ns = row['mtime_ns']
172        entry.st_ctime_ns = row['ctime_ns']
173
174        return entry
175
176    async def readlink(self, inode, ctx):
177        return self.get_row('SELECT * FROM inodes WHERE id=?', (inode,))['target']
178
179    async def opendir(self, inode, ctx):
180        return inode
181
182    async def readdir(self, inode, off, token):
183        if off == 0:
184            off = -1
185
186        cursor2 = self.db.cursor()
187        cursor2.execute("SELECT * FROM contents WHERE parent_inode=? "
188                        'AND rowid > ? ORDER BY rowid', (inode, off))
189
190        for row in cursor2:
191            pyfuse3.readdir_reply(
192                token, row['name'], await self.getattr(row['inode']), row['rowid'])
193
194    async def unlink(self, inode_p, name,ctx):
195        entry = await self.lookup(inode_p, name)
196
197        if stat.S_ISDIR(entry.st_mode):
198            raise pyfuse3.FUSEError(errno.EISDIR)
199
200        self._remove(inode_p, name, entry)
201
202    async def rmdir(self, inode_p, name, ctx):
203        entry = await self.lookup(inode_p, name)
204
205        if not stat.S_ISDIR(entry.st_mode):
206            raise pyfuse3.FUSEError(errno.ENOTDIR)
207
208        self._remove(inode_p, name, entry)
209
210    def _remove(self, inode_p, name, entry):
211        if self.get_row("SELECT COUNT(inode) FROM contents WHERE parent_inode=?",
212                        (entry.st_ino,))[0] > 0:
213            raise pyfuse3.FUSEError(errno.ENOTEMPTY)
214
215        self.cursor.execute("DELETE FROM contents WHERE name=? AND parent_inode=?",
216                        (name, inode_p))
217
218        if entry.st_nlink == 1 and entry.st_ino not in self.inode_open_count:
219            self.cursor.execute("DELETE FROM inodes WHERE id=?", (entry.st_ino,))
220
221    async def symlink(self, inode_p, name, target, ctx):
222        mode = (stat.S_IFLNK | stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR |
223                stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP |
224                stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH)
225        return await self._create(inode_p, name, mode, ctx, target=target)
226
227    async def rename(self, inode_p_old, name_old, inode_p_new, name_new,
228                     flags, ctx):
229        if flags != 0:
230            raise FUSEError(errno.EINVAL)
231
232        entry_old = await self.lookup(inode_p_old, name_old)
233
234        try:
235            entry_new = await self.lookup(inode_p_new, name_new)
236        except pyfuse3.FUSEError as exc:
237            if exc.errno != errno.ENOENT:
238                raise
239            target_exists = False
240        else:
241            target_exists = True
242
243        if target_exists:
244            self._replace(inode_p_old, name_old, inode_p_new, name_new,
245                          entry_old, entry_new)
246        else:
247            self.cursor.execute("UPDATE contents SET name=?, parent_inode=? WHERE name=? "
248                                "AND parent_inode=?", (name_new, inode_p_new,
249                                                       name_old, inode_p_old))
250
251    def _replace(self, inode_p_old, name_old, inode_p_new, name_new,
252                 entry_old, entry_new):
253
254        if self.get_row("SELECT COUNT(inode) FROM contents WHERE parent_inode=?",
255                        (entry_new.st_ino,))[0] > 0:
256            raise pyfuse3.FUSEError(errno.ENOTEMPTY)
257
258        self.cursor.execute("UPDATE contents SET inode=? WHERE name=? AND parent_inode=?",
259                            (entry_old.st_ino, name_new, inode_p_new))
260        self.db.execute('DELETE FROM contents WHERE name=? AND parent_inode=?',
261                        (name_old, inode_p_old))
262
263        if entry_new.st_nlink == 1 and entry_new.st_ino not in self.inode_open_count:
264            self.cursor.execute("DELETE FROM inodes WHERE id=?", (entry_new.st_ino,))
265
266
267    async def link(self, inode, new_inode_p, new_name, ctx):
268        entry_p = await self.getattr(new_inode_p)
269        if entry_p.st_nlink == 0:
270            log.warning('Attempted to create entry %s with unlinked parent %d',
271                        new_name, new_inode_p)
272            raise FUSEError(errno.EINVAL)
273
274        self.cursor.execute("INSERT INTO contents (name, inode, parent_inode) VALUES(?,?,?)",
275                            (new_name, inode, new_inode_p))
276
277        return await self.getattr(inode)
278
279    async def setattr(self, inode, attr, fields, fh, ctx):
280
281        if fields.update_size:
282            data = self.get_row('SELECT data FROM inodes WHERE id=?', (inode,))[0]
283            if data is None:
284                data = b''
285            if len(data) < attr.st_size:
286                data = data + b'\0' * (attr.st_size - len(data))
287            else:
288                data = data[:attr.st_size]
289            self.cursor.execute('UPDATE inodes SET data=?, size=? WHERE id=?',
290                                (memoryview(data), attr.st_size, inode))
291        if fields.update_mode:
292            self.cursor.execute('UPDATE inodes SET mode=? WHERE id=?',
293                                (attr.st_mode, inode))
294
295        if fields.update_uid:
296            self.cursor.execute('UPDATE inodes SET uid=? WHERE id=?',
297                                (attr.st_uid, inode))
298
299        if fields.update_gid:
300            self.cursor.execute('UPDATE inodes SET gid=? WHERE id=?',
301                                (attr.st_gid, inode))
302
303        if fields.update_atime:
304            self.cursor.execute('UPDATE inodes SET atime_ns=? WHERE id=?',
305                                (attr.st_atime_ns, inode))
306
307        if fields.update_mtime:
308            self.cursor.execute('UPDATE inodes SET mtime_ns=? WHERE id=?',
309                                (attr.st_mtime_ns, inode))
310
311        if fields.update_ctime:
312            self.cursor.execute('UPDATE inodes SET ctime_ns=? WHERE id=?',
313                                (attr.st_ctime_ns, inode))
314        else:
315            self.cursor.execute('UPDATE inodes SET ctime_ns=? WHERE id=?',
316                                (int(time()*1e9), inode))
317
318        return await self.getattr(inode)
319
320    async def mknod(self, inode_p, name, mode, rdev, ctx):
321        return await self._create(inode_p, name, mode, ctx, rdev=rdev)
322
323    async def mkdir(self, inode_p, name, mode, ctx):
324        return await self._create(inode_p, name, mode, ctx)
325
326    async def statfs(self, ctx):
327        stat_ = pyfuse3.StatvfsData()
328
329        stat_.f_bsize = 512
330        stat_.f_frsize = 512
331
332        size = self.get_row('SELECT SUM(size) FROM inodes')[0]
333        stat_.f_blocks = size // stat_.f_frsize
334        stat_.f_bfree = max(size // stat_.f_frsize, 1024)
335        stat_.f_bavail = stat_.f_bfree
336
337        inodes = self.get_row('SELECT COUNT(id) FROM inodes')[0]
338        stat_.f_files = inodes
339        stat_.f_ffree = max(inodes , 100)
340        stat_.f_favail = stat_.f_ffree
341
342        return stat_
343
344    async def open(self, inode, flags, ctx):
345        # Yeah, unused arguments
346        #pylint: disable=W0613
347        self.inode_open_count[inode] += 1
348
349        # Use inodes as a file handles
350        return pyfuse3.FileInfo(fh=inode)
351
352    async def access(self, inode, mode, ctx):
353        # Yeah, could be a function and has unused arguments
354        #pylint: disable=R0201,W0613
355        return True
356
357    async def create(self, inode_parent, name, mode, flags, ctx):
358        #pylint: disable=W0612
359        entry = await self._create(inode_parent, name, mode, ctx)
360        self.inode_open_count[entry.st_ino] += 1
361        return (pyfuse3.FileInfo(fh=entry.st_ino), entry)
362
363    async def _create(self, inode_p, name, mode, ctx, rdev=0, target=None):
364        if (await self.getattr(inode_p)).st_nlink == 0:
365            log.warning('Attempted to create entry %s with unlinked parent %d',
366                        name, inode_p)
367            raise FUSEError(errno.EINVAL)
368
369        now_ns = int(time() * 1e9)
370        self.cursor.execute('INSERT INTO inodes (uid, gid, mode, mtime_ns, atime_ns, '
371                            'ctime_ns, target, rdev) VALUES(?, ?, ?, ?, ?, ?, ?, ?)',
372                            (ctx.uid, ctx.gid, mode, now_ns, now_ns, now_ns, target, rdev))
373
374        inode = self.cursor.lastrowid
375        self.db.execute("INSERT INTO contents(name, inode, parent_inode) VALUES(?,?,?)",
376                        (name, inode, inode_p))
377        return await self.getattr(inode)
378
379    async def read(self, fh, offset, length):
380        data = self.get_row('SELECT data FROM inodes WHERE id=?', (fh,))[0]
381        if data is None:
382            data = b''
383        return data[offset:offset+length]
384
385    async def write(self, fh, offset, buf):
386        data = self.get_row('SELECT data FROM inodes WHERE id=?', (fh,))[0]
387        if data is None:
388            data = b''
389        data = data[:offset] + buf + data[offset+len(buf):]
390
391        self.cursor.execute('UPDATE inodes SET data=?, size=? WHERE id=?',
392                            (memoryview(data), len(data), fh))
393        return len(buf)
394
395    async def release(self, fh):
396        self.inode_open_count[fh] -= 1
397
398        if self.inode_open_count[fh] == 0:
399            del self.inode_open_count[fh]
400            if (await self.getattr(fh)).st_nlink == 0:
401                self.cursor.execute("DELETE FROM inodes WHERE id=?", (fh,))
402
403class NoUniqueValueError(Exception):
404    def __str__(self):
405        return 'Query generated more than 1 result row'
406
407
408class NoSuchRowError(Exception):
409    def __str__(self):
410        return 'Query produced 0 result rows'
411
412def init_logging(debug=False):
413    formatter = logging.Formatter('%(asctime)s.%(msecs)03d %(threadName)s: '
414                                  '[%(name)s] %(message)s', datefmt="%Y-%m-%d %H:%M:%S")
415    handler = logging.StreamHandler()
416    handler.setFormatter(formatter)
417    root_logger = logging.getLogger()
418    if debug:
419        handler.setLevel(logging.DEBUG)
420        root_logger.setLevel(logging.DEBUG)
421    else:
422        handler.setLevel(logging.INFO)
423        root_logger.setLevel(logging.INFO)
424    root_logger.addHandler(handler)
425
426def parse_args():
427    '''Parse command line'''
428
429    parser = ArgumentParser()
430
431    parser.add_argument('mountpoint', type=str,
432                        help='Where to mount the file system')
433    parser.add_argument('--debug', action='store_true', default=False,
434                        help='Enable debugging output')
435    parser.add_argument('--debug-fuse', action='store_true', default=False,
436                        help='Enable FUSE debugging output')
437
438    return parser.parse_args()
439
440if __name__ == '__main__':
441
442    options = parse_args()
443    init_logging(options.debug)
444    operations = Operations()
445
446    fuse_options = set(pyfuse3.default_options)
447    fuse_options.add('fsname=tmpfs')
448    fuse_options.discard('default_permissions')
449    if options.debug_fuse:
450        fuse_options.add('debug')
451    pyfuse3.init(operations, options.mountpoint, fuse_options)
452
453    try:
454        trio.run(pyfuse3.main)
455    except:
456        pyfuse3.close(unmount=False)
457        raise
458
459    pyfuse3.close()

Passthrough / Overlay File System

(shipped as examples/passthroughfs.py)

  1#!/usr/bin/env python3
  2'''
  3passthroughfs.py - Example file system for pyfuse3
  4
  5This file system mirrors the contents of a specified directory tree.
  6
  7Caveats:
  8
  9 * Inode generation numbers are not passed through but set to zero.
 10
 11 * Block size (st_blksize) and number of allocated blocks (st_blocks) are not
 12   passed through.
 13
 14 * Performance for large directories is not good, because the directory
 15   is always read completely.
 16
 17 * There may be a way to break-out of the directory tree.
 18
 19 * The readdir implementation is not fully POSIX compliant. If a directory
 20   contains hardlinks and is modified during a readdir call, readdir()
 21   may return some of the hardlinked files twice or omit them completely.
 22
 23 * If you delete or rename files in the underlying file system, the
 24   passthrough file system will get confused.
 25
 26Copyright ©  Nikolaus Rath <Nikolaus.org>
 27
 28Permission is hereby granted, free of charge, to any person obtaining a copy of
 29this software and associated documentation files (the "Software"), to deal in
 30the Software without restriction, including without limitation the rights to
 31use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
 32the Software, and to permit persons to whom the Software is furnished to do so.
 33
 34THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 35IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
 36FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
 37COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
 38IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 39CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 40'''
 41
 42import os
 43import sys
 44
 45# If we are running from the pyfuse3 source directory, try
 46# to load the module from there first.
 47basedir = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '..'))
 48if (os.path.exists(os.path.join(basedir, 'setup.py')) and
 49    os.path.exists(os.path.join(basedir, 'src', 'pyfuse3', '__init__.pyx'))):
 50    sys.path.insert(0, os.path.join(basedir, 'src'))
 51
 52import pyfuse3
 53from argparse import ArgumentParser
 54import errno
 55import logging
 56import stat as stat_m
 57from pyfuse3 import FUSEError
 58from os import fsencode, fsdecode
 59from collections import defaultdict
 60import trio
 61
 62import faulthandler
 63faulthandler.enable()
 64
 65log = logging.getLogger(__name__)
 66
 67class Operations(pyfuse3.Operations):
 68
 69    enable_writeback_cache = True
 70
 71    def __init__(self, source):
 72        super().__init__()
 73        self._inode_path_map = { pyfuse3.ROOT_INODE: source }
 74        self._lookup_cnt = defaultdict(lambda : 0)
 75        self._fd_inode_map = dict()
 76        self._inode_fd_map = dict()
 77        self._fd_open_count = dict()
 78
 79    def _inode_to_path(self, inode):
 80        try:
 81            val = self._inode_path_map[inode]
 82        except KeyError:
 83            raise FUSEError(errno.ENOENT)
 84
 85        if isinstance(val, set):
 86            # In case of hardlinks, pick any path
 87            val = next(iter(val))
 88        return val
 89
 90    def _add_path(self, inode, path):
 91        log.debug('_add_path for %d, %s', inode, path)
 92        self._lookup_cnt[inode] += 1
 93
 94        # With hardlinks, one inode may map to multiple paths.
 95        if inode not in self._inode_path_map:
 96            self._inode_path_map[inode] = path
 97            return
 98
 99        val = self._inode_path_map[inode]
100        if isinstance(val, set):
101            val.add(path)
102        elif val != path:
103            self._inode_path_map[inode] = { path, val }
104
105    async def forget(self, inode_list):
106        for (inode, nlookup) in inode_list:
107            if self._lookup_cnt[inode] > nlookup:
108                self._lookup_cnt[inode] -= nlookup
109                continue
110            log.debug('forgetting about inode %d', inode)
111            assert inode not in self._inode_fd_map
112            del self._lookup_cnt[inode]
113            try:
114                del self._inode_path_map[inode]
115            except KeyError: # may have been deleted
116                pass
117
118    async def lookup(self, inode_p, name, ctx=None):
119        name = fsdecode(name)
120        log.debug('lookup for %s in %d', name, inode_p)
121        path = os.path.join(self._inode_to_path(inode_p), name)
122        attr = self._getattr(path=path)
123        if name != '.' and name != '..':
124            self._add_path(attr.st_ino, path)
125        return attr
126
127    async def getattr(self, inode, ctx=None):
128        if inode in self._inode_fd_map:
129            return self._getattr(fd=self._inode_fd_map[inode])
130        else:
131            return self._getattr(path=self._inode_to_path(inode))
132
133    def _getattr(self, path=None, fd=None):
134        assert fd is None or path is None
135        assert not(fd is None and path is None)
136        try:
137            if fd is None:
138                stat = os.lstat(path)
139            else:
140                stat = os.fstat(fd)
141        except OSError as exc:
142            raise FUSEError(exc.errno)
143
144        entry = pyfuse3.EntryAttributes()
145        for attr in ('st_ino', 'st_mode', 'st_nlink', 'st_uid', 'st_gid',
146                     'st_rdev', 'st_size', 'st_atime_ns', 'st_mtime_ns',
147                     'st_ctime_ns'):
148            setattr(entry, attr, getattr(stat, attr))
149        entry.generation = 0
150        entry.entry_timeout = 0
151        entry.attr_timeout = 0
152        entry.st_blksize = 512
153        entry.st_blocks = ((entry.st_size+entry.st_blksize-1) // entry.st_blksize)
154
155        return entry
156
157    async def readlink(self, inode, ctx):
158        path = self._inode_to_path(inode)
159        try:
160            target = os.readlink(path)
161        except OSError as exc:
162            raise FUSEError(exc.errno)
163        return fsencode(target)
164
165    async def opendir(self, inode, ctx):
166        return inode
167
168    async def readdir(self, inode, off, token):
169        path = self._inode_to_path(inode)
170        log.debug('reading %s', path)
171        entries = []
172        for name in os.listdir(path):
173            if name == '.' or name == '..':
174                continue
175            attr = self._getattr(path=os.path.join(path, name))
176            entries.append((attr.st_ino, name, attr))
177
178        log.debug('read %d entries, starting at %d', len(entries), off)
179
180        # This is not fully posix compatible. If there are hardlinks
181        # (two names with the same inode), we don't have a unique
182        # offset to start in between them. Note that we cannot simply
183        # count entries, because then we would skip over entries
184        # (or return them more than once) if the number of directory
185        # entries changes between two calls to readdir().
186        for (ino, name, attr) in sorted(entries):
187            if ino <= off:
188                continue
189            if not pyfuse3.readdir_reply(
190                token, fsencode(name), attr, ino):
191                break
192            self._add_path(attr.st_ino, os.path.join(path, name))
193
194    async def unlink(self, inode_p, name, ctx):
195        name = fsdecode(name)
196        parent = self._inode_to_path(inode_p)
197        path = os.path.join(parent, name)
198        try:
199            inode = os.lstat(path).st_ino
200            os.unlink(path)
201        except OSError as exc:
202            raise FUSEError(exc.errno)
203        if inode in self._lookup_cnt:
204            self._forget_path(inode, path)
205
206    async def rmdir(self, inode_p, name, ctx):
207        name = fsdecode(name)
208        parent = self._inode_to_path(inode_p)
209        path = os.path.join(parent, name)
210        try:
211            inode = os.lstat(path).st_ino
212            os.rmdir(path)
213        except OSError as exc:
214            raise FUSEError(exc.errno)
215        if inode in self._lookup_cnt:
216            self._forget_path(inode, path)
217
218    def _forget_path(self, inode, path):
219        log.debug('forget %s for %d', path, inode)
220        val = self._inode_path_map[inode]
221        if isinstance(val, set):
222            val.remove(path)
223            if len(val) == 1:
224                self._inode_path_map[inode] = next(iter(val))
225        else:
226            del self._inode_path_map[inode]
227
228    async def symlink(self, inode_p, name, target, ctx):
229        name = fsdecode(name)
230        target = fsdecode(target)
231        parent = self._inode_to_path(inode_p)
232        path = os.path.join(parent, name)
233        try:
234            os.symlink(target, path)
235            os.chown(path, ctx.uid, ctx.gid, follow_symlinks=False)
236        except OSError as exc:
237            raise FUSEError(exc.errno)
238        stat = os.lstat(path)
239        self._add_path(stat.st_ino, path)
240        return await self.getattr(stat.st_ino)
241
242    async def rename(self, inode_p_old, name_old, inode_p_new, name_new,
243                     flags, ctx):
244        if flags != 0:
245            raise FUSEError(errno.EINVAL)
246
247        name_old = fsdecode(name_old)
248        name_new = fsdecode(name_new)
249        parent_old = self._inode_to_path(inode_p_old)
250        parent_new = self._inode_to_path(inode_p_new)
251        path_old = os.path.join(parent_old, name_old)
252        path_new = os.path.join(parent_new, name_new)
253        try:
254            os.rename(path_old, path_new)
255            inode = os.lstat(path_new).st_ino
256        except OSError as exc:
257            raise FUSEError(exc.errno)
258        if inode not in self._lookup_cnt:
259            return
260
261        val = self._inode_path_map[inode]
262        if isinstance(val, set):
263            assert len(val) > 1
264            val.add(path_new)
265            val.remove(path_old)
266        else:
267            assert val == path_old
268            self._inode_path_map[inode] = path_new
269
270    async def link(self, inode, new_inode_p, new_name, ctx):
271        new_name = fsdecode(new_name)
272        parent = self._inode_to_path(new_inode_p)
273        path = os.path.join(parent, new_name)
274        try:
275            os.link(self._inode_to_path(inode), path, follow_symlinks=False)
276        except OSError as exc:
277            raise FUSEError(exc.errno)
278        self._add_path(inode, path)
279        return await self.getattr(inode)
280
281    async def setattr(self, inode, attr, fields, fh, ctx):
282        # We use the f* functions if possible so that we can handle
283        # a setattr() call for an inode without associated directory
284        # handle.
285        if fh is None:
286            path_or_fh = self._inode_to_path(inode)
287            truncate = os.truncate
288            chmod = os.chmod
289            chown = os.chown
290            stat = os.lstat
291        else:
292            path_or_fh = fh
293            truncate = os.ftruncate
294            chmod = os.fchmod
295            chown = os.fchown
296            stat = os.fstat
297
298        try:
299            if fields.update_size:
300                truncate(path_or_fh, attr.st_size)
301
302            if fields.update_mode:
303                # Under Linux, chmod always resolves symlinks so we should
304                # actually never get a setattr() request for a symbolic
305                # link.
306                assert not stat_m.S_ISLNK(attr.st_mode)
307                chmod(path_or_fh, stat_m.S_IMODE(attr.st_mode))
308
309            if fields.update_uid:
310                chown(path_or_fh, attr.st_uid, -1, follow_symlinks=False)
311
312            if fields.update_gid:
313                chown(path_or_fh, -1, attr.st_gid, follow_symlinks=False)
314
315            if fields.update_atime and fields.update_mtime:
316                if fh is None:
317                    os.utime(path_or_fh, None, follow_symlinks=False,
318                             ns=(attr.st_atime_ns, attr.st_mtime_ns))
319                else:
320                    os.utime(path_or_fh, None,
321                             ns=(attr.st_atime_ns, attr.st_mtime_ns))
322            elif fields.update_atime or fields.update_mtime:
323                # We can only set both values, so we first need to retrieve the
324                # one that we shouldn't be changing.
325                oldstat = stat(path_or_fh)
326                if not fields.update_atime:
327                    attr.st_atime_ns = oldstat.st_atime_ns
328                else:
329                    attr.st_mtime_ns = oldstat.st_mtime_ns
330                if fh is None:
331                    os.utime(path_or_fh, None, follow_symlinks=False,
332                             ns=(attr.st_atime_ns, attr.st_mtime_ns))
333                else:
334                    os.utime(path_or_fh, None,
335                             ns=(attr.st_atime_ns, attr.st_mtime_ns))
336
337        except OSError as exc:
338            raise FUSEError(exc.errno)
339
340        return await self.getattr(inode)
341
342    async def mknod(self, inode_p, name, mode, rdev, ctx):
343        path = os.path.join(self._inode_to_path(inode_p), fsdecode(name))
344        try:
345            os.mknod(path, mode=(mode & ~ctx.umask), device=rdev)
346            os.chown(path, ctx.uid, ctx.gid)
347        except OSError as exc:
348            raise FUSEError(exc.errno)
349        attr = self._getattr(path=path)
350        self._add_path(attr.st_ino, path)
351        return attr
352
353    async def mkdir(self, inode_p, name, mode, ctx):
354        path = os.path.join(self._inode_to_path(inode_p), fsdecode(name))
355        try:
356            os.mkdir(path, mode=(mode & ~ctx.umask))
357            os.chown(path, ctx.uid, ctx.gid)
358        except OSError as exc:
359            raise FUSEError(exc.errno)
360        attr = self._getattr(path=path)
361        self._add_path(attr.st_ino, path)
362        return attr
363
364    async def statfs(self, ctx):
365        root = self._inode_path_map[pyfuse3.ROOT_INODE]
366        stat_ = pyfuse3.StatvfsData()
367        try:
368            statfs = os.statvfs(root)
369        except OSError as exc:
370            raise FUSEError(exc.errno)
371        for attr in ('f_bsize', 'f_frsize', 'f_blocks', 'f_bfree', 'f_bavail',
372                     'f_files', 'f_ffree', 'f_favail'):
373            setattr(stat_, attr, getattr(statfs, attr))
374        stat_.f_namemax = statfs.f_namemax - (len(root)+1)
375        return stat_
376
377    async def open(self, inode, flags, ctx):
378        if inode in self._inode_fd_map:
379            fd = self._inode_fd_map[inode]
380            self._fd_open_count[fd] += 1
381            return pyfuse3.FileInfo(fh=fd)
382        assert flags & os.O_CREAT == 0
383        try:
384            fd = os.open(self._inode_to_path(inode), flags)
385        except OSError as exc:
386            raise FUSEError(exc.errno)
387        self._inode_fd_map[inode] = fd
388        self._fd_inode_map[fd] = inode
389        self._fd_open_count[fd] = 1
390        return pyfuse3.FileInfo(fh=fd)
391
392    async def create(self, inode_p, name, mode, flags, ctx):
393        path = os.path.join(self._inode_to_path(inode_p), fsdecode(name))
394        try:
395            fd = os.open(path, flags | os.O_CREAT | os.O_TRUNC)
396        except OSError as exc:
397            raise FUSEError(exc.errno)
398        attr = self._getattr(fd=fd)
399        self._add_path(attr.st_ino, path)
400        self._inode_fd_map[attr.st_ino] = fd
401        self._fd_inode_map[fd] = attr.st_ino
402        self._fd_open_count[fd] = 1
403        return (pyfuse3.FileInfo(fh=fd), attr)
404
405    async def read(self, fd, offset, length):
406        os.lseek(fd, offset, os.SEEK_SET)
407        return os.read(fd, length)
408
409    async def write(self, fd, offset, buf):
410        os.lseek(fd, offset, os.SEEK_SET)
411        return os.write(fd, buf)
412
413    async def release(self, fd):
414        if self._fd_open_count[fd] > 1:
415            self._fd_open_count[fd] -= 1
416            return
417
418        del self._fd_open_count[fd]
419        inode = self._fd_inode_map[fd]
420        del self._inode_fd_map[inode]
421        del self._fd_inode_map[fd]
422        try:
423            os.close(fd)
424        except OSError as exc:
425            raise FUSEError(exc.errno)
426
427def init_logging(debug=False):
428    formatter = logging.Formatter('%(asctime)s.%(msecs)03d %(threadName)s: '
429                                  '[%(name)s] %(message)s', datefmt="%Y-%m-%d %H:%M:%S")
430    handler = logging.StreamHandler()
431    handler.setFormatter(formatter)
432    root_logger = logging.getLogger()
433    if debug:
434        handler.setLevel(logging.DEBUG)
435        root_logger.setLevel(logging.DEBUG)
436    else:
437        handler.setLevel(logging.INFO)
438        root_logger.setLevel(logging.INFO)
439    root_logger.addHandler(handler)
440
441
442def parse_args(args):
443    '''Parse command line'''
444
445    parser = ArgumentParser()
446
447    parser.add_argument('source', type=str,
448                        help='Directory tree to mirror')
449    parser.add_argument('mountpoint', type=str,
450                        help='Where to mount the file system')
451    parser.add_argument('--debug', action='store_true', default=False,
452                        help='Enable debugging output')
453    parser.add_argument('--debug-fuse', action='store_true', default=False,
454                        help='Enable FUSE debugging output')
455
456    return parser.parse_args(args)
457
458def main():
459    options = parse_args(sys.argv[1:])
460    init_logging(options.debug)
461    operations = Operations(options.source)
462
463    log.debug('Mounting...')
464    fuse_options = set(pyfuse3.default_options)
465    fuse_options.add('fsname=passthroughfs')
466    if options.debug_fuse:
467        fuse_options.add('debug')
468    pyfuse3.init(operations, options.mountpoint, fuse_options)
469
470    try:
471        log.debug('Entering main loop..')
472        trio.run(pyfuse3.main)
473    except:
474        pyfuse3.close(unmount=False)
475        raise
476
477    log.debug('Unmounting..')
478    pyfuse3.close()
479
480if __name__ == '__main__':
481    main()