Components/IFS/src/include/IFS/FWFS/Object.h
Go to the documentation of this file.
1 /****
2  * Object.h
3  * Basic definitions for FW file system structure.
4  *
5  * Created on: 7 Aug 2018
6  *
7  * Copyright 2019 mikee47 <mike@sillyhouse.net>
8  *
9  * This file is part of the IFS Library
10  *
11  * This library is free software: you can redistribute it and/or modify it under the terms of the
12  * GNU General Public License as published by the Free Software Foundation, version 3 or later.
13  *
14  * This library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
15  * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
16  * See the GNU General Public License for more details.
17  *
18  * You should have received a copy of the GNU General Public License along with this library.
19  * If not, see <https://www.gnu.org/licenses/>.
20  *
21  *
22  * A filesystem image is basically:
23  *
24  * uint32_t START_MARKER;
25  * Object Objects[];
26  * Object EndObject;
27  * uint32_t END_MARKER;
28  *
29  * An FWFS Object is a variable-length structure which can be read as either 1 or 2 words,
30  * depending on the type.
31  * Objects always start on a word boundary.
32  * File and directory objects are both emitted as 'named' objects which contain a list of
33  * zero or more child objects. When the image is built the child objects are emitted first,
34  * followed by the parent. This puts the root directory at the end of the image.
35  * This arrangement means an image can be generated without having to back-track and rewrite
36  * headers.
37  *
38  * Child objects can be of any type. A directory object will mostly contain other file and
39  * directory objects. File data is stored in a data object, not in the file object itself.
40  * This is usually found following the file object, but it doesn't have to be.
41  * Any object may be referenced by zero or more named object(s). For example, file links/aliases
42  * can contain references to the same file data.
43  * A file object may contain multiple data objects. These are treated as a contiguous block for file operations.
44  * This would potentially allow a file system builder to place common file blocks into shared data objects.
45  *
46  * Object names are from 0 to 255 characters, inclusive. The root directory has a zero-length name.
47  * Paths lengths are unlimited.
48  * '/' is used as the path separator. It informs the filesystem of the parent/child relationship
49  * between one directory object and a sub-ordinate.
50  * ':' is used as the file stream separator. It performs the equivalent of the path separator for
51  * non-directory named objects. For exaample, file object may contain named objects accessible thus:
52  * index.html:info opens a handle to a named object called 'info' belonging to index.html.
53  *
54  * OK, I admit it; this is pinched from NTFS; but it's such a cool idea. Applications can use it
55  * to attach custom data to their files without limitation.
56  *
57  * As a side note, of course FWFS is read-only. More precisely, it only supports random reading
58  * of files, not random writing. Serial writing is supported in the form of image creation.
59  *
60  * For SPIFFS, IFS is a wrapper. The metadata features are supported using SPIFFS metadata.
61  * An alternative approach is to implement every named object as a SPIFFS file. We'd then get
62  * all the features of FWFS in a rewritable system, with all the benefits of SPIFFS wear-levelling.
63  *
64  * Objects are identified by their index, but it's not stored in the image. Instead, it's
65  * tracked via internal object descriptor.
66  *
67  * To optimise lookups, an object table can be stored at the end of the image. This is
68  * just an array of 32-bit image offsets so that an object can be located instantly on
69  * large volumes. This will be optional as it can consume significant space.
70  *
71  ****/
72 
73 #pragma once
74 
75 #include "../TimeStamp.h"
76 #include "../FileAttributes.h"
77 #include "../Compression.h"
78 #include "../UserRole.h"
79 
80 namespace IFS
81 {
82 namespace FWFS
83 {
84 /*
85  * Helper template function to get the next object pointer at (current + offset)
86  * casting as required for the given return type.
87  */
88 template <typename T> static T at_offset(const void* current, int offset)
89 {
90  auto p = reinterpret_cast<const uint8_t*>(current) + offset;
91  return reinterpret_cast<T>(p);
92 }
93 
94 template <typename T> static T at_offset(void* current, int offset)
95 {
96  auto p = reinterpret_cast<uint8_t*>(current) + offset;
97  return reinterpret_cast<T>(p);
98 }
99 
100 // First object located immediately after start marker in image
101 constexpr size_t FWFS_BASE_OFFSET{sizeof(uint32_t)};
102 
103 // Images start with a single word to identify this as a Firmware Filesystem image
104 constexpr uint32_t FWFILESYS_START_MARKER{0x53465746}; // "FWFS"
105 
106 // Image end marker (reverse of magic)
107 constexpr uint32_t FWFILESYS_END_MARKER{0x46574653}; // "SFWF"
108 
109 // Everything in this header must be portable (at least, with other little-endian systems) so byte-align and pack it
110 #pragma pack(1)
111 
119 #define FWFS_OBJTYPE_MAP(XX) \
120  XX(0, End, "The system image footer") \
121  XX(1, Data8, "Data, max 255 bytes") \
122  XX(2, ID32, "32-bit identifier") \
123  XX(3, ObjAttr, "Object attributes") \
124  XX(4, Compression, "Compression descriptor") \
125  XX(5, ReadACE, "minimum UserRole for read access") \
126  XX(6, WriteACE, "minimum UserRole for write access") \
127  XX(7, VolumeIndex, "Volume index number") \
128  XX(8, Md5Hash, "MD5 Hash Value") \
129  XX(9, Comment, "Comment") \
130  XX(10, UserAttribute, "User Attribute") \
131  XX(32, Data16, "Data, max 64K - 1") \
132  XX(33, Volume, "Volume, top-level container object") \
133  XX(34, MountPoint, "Root for another filesystem") \
134  XX(35, Directory, "Directory entry") \
135  XX(36, File, "File entry") \
136  XX(64, Data24, "Data, max 16M - 1")
137 
144 struct Object {
145  uint8_t typeData;
146 
150  using ID = uint32_t;
151 
152  enum class Type {
153 #define XX(value, tag, text) tag = value,
155 #undef XX
156  };
157 
158  // Top bit of object type set to indicate a reference
159  static constexpr uint8_t FWOBT_REF{0x80};
160 
165  enum class Attribute {
169  ReadOnly,
173  Archive,
180  Encrypted,
181  //
182  MAX
183  };
184 
185  using Attributes = BitSet<uint8_t, Attribute, size_t(Attribute::MAX)>;
186 
187  Type type() const
188  {
189  return static_cast<Type>(typeData & 0x7f);
190  }
191 
192  void setType(Type type, bool isRef = false)
193  {
195  if(isRef) {
196  typeData |= FWOBT_REF;
197  }
198  }
199 
200  bool isRef() const
201  {
202  return (typeData & FWOBT_REF) != 0;
203  }
204 
205  uint32_t getRef() const
206  {
207  if(!isRef()) {
208  return 0;
209  }
210  uint32_t mask = 0xffffffff >> ((4 - data8.contentSize()) * 8);
211  return data8.ref.packedOffset & mask;
212  }
213 
214  bool isNamed() const
215  {
216  return type() >= Type::Volume && type() < Type::Data24;
217  }
218 
219  bool isData() const
220  {
221  return type() == Type::Data8 || type() == Type::Data16 || type() == Type::Data24;
222  }
223 
224  bool isDir() const
225  {
226  return type() == Type::Directory;
227  }
228 
229  bool isMountPoint() const
230  {
231  return type() == Type::MountPoint;
232  }
233 
234  union {
235  // Data8 - 8-bit size, max. 255 bytes
236  struct {
237  uint8_t _contentSize;
238  // uint8_t data[];
239 
240  unsigned contentOffset() const
241  {
242  return offsetof(Object, data8) + 1;
243  }
244 
245  unsigned contentSize() const
246  {
247  return _contentSize;
248  }
249 
250  void setContentSize(uint32_t value)
251  {
253  }
254 
255  union {
256  // An object reference: the contents are stored externally (on the same volume)
257  struct {
258  uint32_t packedOffset; // 1-4 byte offset
259  } ref;
260 
261  // ID32
262  struct {
263  uint32_t value;
264  } id32;
265 
266  // Object attributes
267  struct {
268  uint8_t attr; // Attributes
270 
271  // Compression descriptor
273 
274  // ReadACE, WriteACE
275  struct {
277  } ace;
278 
279  // Identifies a volume index, contained in a mount point
280  struct {
281  uint8_t index;
283 
284  // User attribute
285  struct {
286  uint8_t tagValue;
287  // uint8_t[] data;
289 
290  // END - immediately followed by end marker
291  struct {
292  uint32_t checksum;
293  } end;
294  };
295  } data8;
296 
297  // Data16 - 16-bit size, max. (64KB - 1)
298  struct {
299  uint16_t _contentSize;
300  // uint8_t data[];
301 
302  unsigned contentOffset() const
303  {
304  return offsetof(Object, data16) + 2;
305  }
306 
307  uint32_t contentSize() const
308  {
309  return _contentSize;
310  }
311 
312  void setContentSize(uint32_t value)
313  {
315  }
316 
317  uint32_t size() const
318  {
319  return contentOffset() + contentSize();
320  }
321 
322  union {
323  /*
324  * named object
325  *
326  * child objects can be contained or referenced (flag in _id)
327  * object attributes are optional so can be another object/attribute
328  */
329  struct {
330  uint8_t namelen;
331  TimeStamp mtime; // Object modification time
332  // char name[namelen];
333  // Object children[];
334 
335  // Offset to object name relative to content start
336  unsigned nameOffset() const
337  {
338  return sizeof(Object::data16.named);
339  }
340 
341  // Offset to start of child object table
342  unsigned childTableOffset() const
343  {
344  return nameOffset() + namelen;
345  }
346 
347  } named;
348  };
350 
351  // Data24 - 24-bit size, max. (16MB - 1)
352  struct {
353  uint16_t _contentSize;
355  // uint8_t data[];
356 
357  size_t contentOffset() const
358  {
359  return offsetof(Object, data24) + 3;
360  }
361 
362  uint32_t contentSize() const
363  {
364  return (_contentSizeHigh << 16) | _contentSize;
365  }
366 
367  void setContentSize(uint32_t value)
368  {
369  _contentSize = value & 0xffff;
370  _contentSizeHigh = value >> 16;
371  }
372 
373  uint32_t size() const
374  {
375  return contentOffset() + contentSize();
376  }
378  };
379 
382  size_t contentOffset() const
383  {
384  if(isRef() || (type() < Type::Data16)) {
385  return data8.contentOffset();
386  } else if(type() < Type::Data24) {
387  return data16.contentOffset();
388  } else {
389  return data24.contentOffset();
390  }
391  }
392 
397  uint32_t contentSize() const
398  {
399  if(isRef() || (type() < Type::Data16)) {
400  return data8.contentSize();
401  } else if(type() < Type::Data24) {
402  return data16.contentSize();
403  } else {
404  return data24.contentSize();
405  }
406  }
407 
408  void setContentSize(size_t size)
409  {
410  if(isRef() || type() < Type::Data16) {
411  data8.setContentSize(size);
412  } else if(type() < Type::Data24) {
413  data16.setContentSize(size);
414  } else {
415  data24.setContentSize(size);
416  }
417  }
418 
419  uint32_t childTableOffset() const
420  {
421  assert(isNamed());
422  return data16.contentOffset() + data16.named.childTableOffset();
423  }
424 
425  uint32_t childTableSize() const
426  {
427  assert(isNamed());
428  int size = data16.contentSize() - data16.named.childTableOffset();
429  return (size <= 0) ? 0 : size;
430  }
431 
435  uint32_t size() const
436  {
437  return contentOffset() + contentSize();
438  }
439 };
440 
441 static_assert(sizeof(Object) == 8, "Object alignment wrong!");
442 
443 #pragma pack()
444 
448 struct FWObjDesc {
451 
452  FWObjDesc(Object::ID objId = 0) : id(objId)
453  {
454  }
455 
456  uint32_t offset() const
457  {
458  return id;
459  }
460 
461  uint32_t nextOffset() const
462  {
463  return offset() + obj.size();
464  }
465 
466  // Move to next object location
467  void next()
468  {
469  id = nextOffset();
470  }
471 
472  uint32_t contentOffset() const
473  {
474  return offset() + obj.contentOffset();
475  }
476 
477  uint32_t childTableOffset() const
478  {
479  return offset() + obj.childTableOffset();
480  }
481 };
482 
485 
486 } // namespace FWFS
487 } // namespace IFS
488 
String toString(IFS::FWFS::Object::Type obt)
Get descriptive String for an object type.
#define FWFS_OBJTYPE_MAP(XX)
Object type identifiers.
Definition: Components/IFS/src/include/IFS/FWFS/Object.h:119
The String class.
Definition: WString.h:137
IFS::Directory Directory
Definition: Core/FileSystem.h:33
Object::Attributes getObjectAttributes(FileAttributes fileAttr)
constexpr uint32_t FWFILESYS_END_MARKER
Definition: Components/IFS/src/include/IFS/FWFS/Object.h:107
FileAttributes getFileAttributes(Object::Attributes objattr)
static T at_offset(const void *current, int offset)
Definition: Components/IFS/src/include/IFS/FWFS/Object.h:88
constexpr size_t FWFS_BASE_OFFSET
Definition: Components/IFS/src/include/IFS/FWFS/Object.h:101
constexpr uint32_t FWFILESYS_START_MARKER
Definition: Components/IFS/src/include/IFS/FWFS/Object.h:104
Definition: DirectoryTemplate.h:37
UserRole
Definition: UserRole.h:36
A compression descriptor.
Definition: Compression.h:38
FWFS Object Descriptor.
Definition: Components/IFS/src/include/IFS/FWFS/Object.h:448
void next()
Definition: Components/IFS/src/include/IFS/FWFS/Object.h:467
uint32_t childTableOffset() const
Definition: Components/IFS/src/include/IFS/FWFS/Object.h:477
Object obj
The object structure.
Definition: Components/IFS/src/include/IFS/FWFS/Object.h:450
uint32_t contentOffset() const
Definition: Components/IFS/src/include/IFS/FWFS/Object.h:472
uint32_t offset() const
Definition: Components/IFS/src/include/IFS/FWFS/Object.h:456
Object::ID id
location
Definition: Components/IFS/src/include/IFS/FWFS/Object.h:449
uint32_t nextOffset() const
Definition: Components/IFS/src/include/IFS/FWFS/Object.h:461
FWObjDesc(Object::ID objId=0)
Definition: Components/IFS/src/include/IFS/FWFS/Object.h:452
Object structure.
Definition: Components/IFS/src/include/IFS/FWFS/Object.h:144
struct IFS::FWFS::Object::@26::@28::@31::@34 id32
Compression compression
Definition: Components/IFS/src/include/IFS/FWFS/Object.h:272
bool isData() const
Definition: Components/IFS/src/include/IFS/FWFS/Object.h:219
bool isNamed() const
Definition: Components/IFS/src/include/IFS/FWFS/Object.h:214
void setContentSize(size_t size)
Definition: Components/IFS/src/include/IFS/FWFS/Object.h:408
struct IFS::FWFS::Object::@26::@28::@31::@36 ace
Attribute
Object attributes.
Definition: Components/IFS/src/include/IFS/FWFS/Object.h:165
@ ReadOnly
Object should not be modified or deleted.
@ Encrypted
Object data is encrypted.
@ Archive
Object modified flag.
struct IFS::FWFS::Object::@26::@28::@31::@33 ref
uint16_t _contentSize
Object size (excluding this header)
Definition: Components/IFS/src/include/IFS/FWFS/Object.h:299
UserRole role
Definition: Components/IFS/src/include/IFS/FWFS/Object.h:276
Type type() const
Definition: Components/IFS/src/include/IFS/FWFS/Object.h:187
struct IFS::FWFS::Object::@26::@28::@31::@39 end
uint32_t value
32-bit identifier, e.g. volume ID
Definition: Components/IFS/src/include/IFS/FWFS/Object.h:263
struct IFS::FWFS::Object::@26::@28 data8
bool isRef() const
Definition: Components/IFS/src/include/IFS/FWFS/Object.h:200
bool isDir() const
Definition: Components/IFS/src/include/IFS/FWFS/Object.h:224
struct IFS::FWFS::Object::@26::@29 data16
uint32_t size() const
total size this object occupies in the image
Definition: Components/IFS/src/include/IFS/FWFS/Object.h:435
uint32_t packedOffset
Definition: Components/IFS/src/include/IFS/FWFS/Object.h:258
struct IFS::FWFS::Object::@26::@28::@31::@38 userAttribute
uint8_t _contentSizeHigh
Allows data up to 16MByte.
Definition: Components/IFS/src/include/IFS/FWFS/Object.h:354
struct IFS::FWFS::Object::@26::@30 data24
uint32_t childTableOffset() const
Definition: Components/IFS/src/include/IFS/FWFS/Object.h:419
bool isMountPoint() const
Definition: Components/IFS/src/include/IFS/FWFS/Object.h:229
uint32_t childTableSize() const
Definition: Components/IFS/src/include/IFS/FWFS/Object.h:425
uint32_t checksum
Definition: Components/IFS/src/include/IFS/FWFS/Object.h:292
struct IFS::FWFS::Object::@26::@28::@31::@35 objectAttributes
Type
Definition: Components/IFS/src/include/IFS/FWFS/Object.h:152
XX(value, tag, text)
uint8_t index
Definition: Components/IFS/src/include/IFS/FWFS/Object.h:281
uint8_t typeData
Stored type plus flag.
Definition: Components/IFS/src/include/IFS/FWFS/Object.h:145
uint32_t contentSize() const
return size of object content, excluding header and size fields
Definition: Components/IFS/src/include/IFS/FWFS/Object.h:397
uint8_t tagValue
Definition: Components/IFS/src/include/IFS/FWFS/Object.h:286
uint8_t namelen
Length of object name.
Definition: Components/IFS/src/include/IFS/FWFS/Object.h:330
struct IFS::FWFS::Object::@26::@28::@31::@37 volumeIndex
void setType(Type type, bool isRef=false)
Definition: Components/IFS/src/include/IFS/FWFS/Object.h:192
TimeStamp mtime
Definition: Components/IFS/src/include/IFS/FWFS/Object.h:331
uint32_t ID
Object identifier (offset from start of image)
Definition: Components/IFS/src/include/IFS/FWFS/Object.h:150
uint8_t _contentSize
Definition: Components/IFS/src/include/IFS/FWFS/Object.h:237
static constexpr uint8_t FWOBT_REF
Definition: Components/IFS/src/include/IFS/FWFS/Object.h:159
uint32_t getRef() const
Definition: Components/IFS/src/include/IFS/FWFS/Object.h:205
size_t contentOffset() const
return offset to start of object content
Definition: Components/IFS/src/include/IFS/FWFS/Object.h:382
uint8_t attr
Definition: Components/IFS/src/include/IFS/FWFS/Object.h:268
struct IFS::FWFS::Object::@26::@29::@40::@42 named
Manage IFS timestamps stored as an unsigned 32-bit value.
Definition: TimeStamp.h:37