FAT on SPI Flash Library (#1809)

* First commit of FFat library

* Fixed reboot loops if no fat present. Added CMakeLists

* Functionalize the partition checks

* Cleanup, especially in format

* Dont format if mounted.  More wording cleanup

* 16M ffat should only be on 16M board

* Fix some casting issues that trip up the compiler when building as ESP-IDF component

CMakeLists.txt

@@ -40,6 +40,7 @@ set(LIBRARY_SRCS
   libraries/DNSServer/src/DNSServer.cpp
   libraries/EEPROM/EEPROM.cpp
   libraries/ESPmDNS/src/ESPmDNS.cpp
+  libraries/FFat/src/FFat.cpp
   libraries/FS/src/FS.cpp
   libraries/FS/src/vfs_api.cpp
   libraries/HTTPClient/src/HTTPClient.cpp
@@ -175,6 +176,7 @@ set(COMPONENT_ADD_INCLUDEDIRS
   libraries/DNSServer/src
   libraries/ESP32/src
   libraries/ESPmDNS/src
+  libraries/FFat/src
   libraries/FS/src
   libraries/HTTPClient/src
   libraries/NetBIOS/src

boards.txt

@@ -46,6 +46,8 @@ esp32.menu.PartitionScheme.no_ota.upload.maximum_size=2097152
 esp32.menu.PartitionScheme.min_spiffs=Minimal SPIFFS (Large APPS with OTA)
 esp32.menu.PartitionScheme.min_spiffs.build.partitions=min_spiffs
 esp32.menu.PartitionScheme.min_spiffs.upload.maximum_size=1966080
+esp32.menu.PartitionScheme.fatflash=16M Fat
+esp32.menu.PartitionScheme.fatflash.build.partitions=ffat
 
 esp32.menu.FlashMode.qio=QIO
 esp32.menu.FlashMode.qio.build.flash_mode=dio
@@ -70,6 +72,9 @@ esp32.menu.FlashSize.4M.build.flash_size=4MB
 esp32.menu.FlashSize.2M=2MB (16Mb)
 esp32.menu.FlashSize.2M.build.flash_size=2MB
 esp32.menu.FlashSize.2M.build.partitions=minimal
+esp32.menu.FlashSize.16M=16MB (128Mb)
+esp32.menu.FlashSize.16M.build.flash_size=16MB
+esp32.menu.FlashSize.16M.build.partitions=ffat
 
 esp32.menu.UploadSpeed.921600=921600
 esp32.menu.UploadSpeed.921600.upload.speed=921600

libraries/FFat/examples/FFat_Test/FFat_Test.ino

@@ -0,0 +1,181 @@
+#include "FS.h"
+#include "FFat.h"
+
+// You only need to format FFat the first time you run a test
+#define FORMAT_FFAT true
+
+void listDir(fs::FS &fs, const char * dirname, uint8_t levels){
+    Serial.printf("Listing directory: %s\r\n", dirname);
+
+    File root = fs.open(dirname);
+    if(!root){
+        Serial.println("- failed to open directory");
+        return;
+    }
+    if(!root.isDirectory()){
+        Serial.println(" - not a directory");
+        return;
+    }
+
+    File file = root.openNextFile();
+    while(file){
+        if(file.isDirectory()){
+            Serial.print("  DIR : ");
+            Serial.println(file.name());
+            if(levels){
+                listDir(fs, file.name(), levels -1);
+            }
+        } else {
+            Serial.print("  FILE: ");
+            Serial.print(file.name());
+            Serial.print("\tSIZE: ");
+            Serial.println(file.size());
+        }
+        file = root.openNextFile();
+    }
+}
+
+void readFile(fs::FS &fs, const char * path){
+    Serial.printf("Reading file: %s\r\n", path);
+
+    File file = fs.open(path);
+    if(!file || file.isDirectory()){
+        Serial.println("- failed to open file for reading");
+        return;
+    }
+
+    Serial.println("- read from file:");
+    while(file.available()){
+        Serial.write(file.read());
+    }
+}
+
+void writeFile(fs::FS &fs, const char * path, const char * message){
+    Serial.printf("Writing file: %s\r\n", path);
+
+    File file = fs.open(path, FILE_WRITE);
+    if(!file){
+        Serial.println("- failed to open file for writing");
+        return;
+    }
+    if(file.print(message)){
+        Serial.println("- file written");
+    } else {
+        Serial.println("- frite failed");
+    }
+}
+
+void appendFile(fs::FS &fs, const char * path, const char * message){
+    Serial.printf("Appending to file: %s\r\n", path);
+
+    File file = fs.open(path, FILE_APPEND);
+    if(!file){
+        Serial.println("- failed to open file for appending");
+        return;
+    }
+    if(file.print(message)){
+        Serial.println("- message appended");
+    } else {
+        Serial.println("- append failed");
+    }
+}
+
+void renameFile(fs::FS &fs, const char * path1, const char * path2){
+    Serial.printf("Renaming file %s to %s\r\n", path1, path2);
+    if (fs.rename(path1, path2)) {
+        Serial.println("- file renamed");
+    } else {
+        Serial.println("- rename failed");
+    }
+}
+
+void deleteFile(fs::FS &fs, const char * path){
+    Serial.printf("Deleting file: %s\r\n", path);
+    if(fs.remove(path)){
+        Serial.println("- file deleted");
+    } else {
+        Serial.println("- delete failed");
+    }
+}
+
+void testFileIO(fs::FS &fs, const char * path){
+    Serial.printf("Testing file I/O with %s\r\n", path);
+
+    static uint8_t buf[512];
+    size_t len = 0;
+    File file = fs.open(path, FILE_WRITE);
+    if(!file){
+        Serial.println("- failed to open file for writing");
+        return;
+    }
+
+    size_t i;
+    Serial.print("- writing" );
+    uint32_t start = millis();
+    for(i=0; i<2048; i++){
+        if ((i & 0x001F) == 0x001F){
+          Serial.print(".");
+        }
+        file.write(buf, 512);
+    }
+    Serial.println("");
+    uint32_t end = millis() - start;
+    Serial.printf(" - %u bytes written in %u ms\r\n", 2048 * 512, end);
+    file.close();
+
+    file = fs.open(path);
+    start = millis();
+    end = start;
+    i = 0;
+    if(file && !file.isDirectory()){
+        len = file.size();
+        size_t flen = len;
+        start = millis();
+        Serial.print("- reading" );
+        while(len){
+            size_t toRead = len;
+            if(toRead > 512){
+                toRead = 512;
+            }
+            file.read(buf, toRead);
+            if ((i++ & 0x001F) == 0x001F){
+              Serial.print(".");
+            }
+            len -= toRead;
+        }
+        Serial.println("");
+        end = millis() - start;
+        Serial.printf("- %u bytes read in %u ms\r\n", flen, end);
+        file.close();
+    } else {
+        Serial.println("- failed to open file for reading");
+    }
+}
+
+void setup(){
+    Serial.begin(115200);
+    Serial.setDebugOutput(true);
+    if (FORMAT_FFAT) FFat.format();
+    if(!FFat.begin()){
+        Serial.println("FFat Mount Failed");
+        return;
+    }
+    
+    Serial.printf("Total space: %10lu\n", FFat.totalBytes());
+    Serial.printf("Free space: %10lu\n", FFat.freeBytes());
+    listDir(FFat, "/", 0);
+    writeFile(FFat, "/hello.txt", "Hello ");
+    appendFile(FFat, "/hello.txt", "World!\r\n");
+    readFile(FFat, "/hello.txt");
+    renameFile(FFat, "/hello.txt", "/foo.txt");
+    readFile(FFat, "/foo.txt");
+    deleteFile(FFat, "/foo.txt");
+    testFileIO(FFat, "/test.txt");
+    Serial.printf("Free space: %10lu\n", FFat.freeBytes());
+    deleteFile(FFat, "/test.txt");
+    Serial.println( "Test complete" );
+}
+
+void loop(){
+
+}

libraries/FFat/library.properties

@@ -0,0 +1,9 @@
+name=FFat
+version=1.0
+author=Hristo Gochkov, Ivan Grokhtkov, Larry Bernstone
+maintainer=Hristo Gochkov <hristo@espressif.com>
+sentence=ESP32 FAT on Flash File System
+paragraph=
+category=Data Storage
+url=
+architectures=esp32

libraries/FFat/src/FFat.cpp

@@ -0,0 +1,144 @@
+// Copyright 2015-2016 Espressif Systems (Shanghai) PTE LTD
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "vfs_api.h"
+extern "C" {
+#include "esp_vfs_fat.h"
+#include "diskio.h"
+#include "diskio_wl.h"
+#include "vfs_fat_internal.h"
+}
+#include "FFat.h"
+
+using namespace fs;
+
+F_Fat::F_Fat(FSImplPtr impl)
+    : FS(impl)
+{}
+
+const esp_partition_t *check_ffat_partition(const char* label)
+{
+    const esp_partition_t* ck_part = esp_partition_find_first(
+       ESP_PARTITION_TYPE_DATA, ESP_PARTITION_SUBTYPE_DATA_FAT, label);
+    if (!ck_part) {
+        log_e("No FAT partition found with label %s", label);
+        return NULL;
+    }
+    return ck_part;
+}
+
+bool F_Fat::begin(bool formatOnFail, const char * basePath, uint8_t maxOpenFiles, const char * partitionLabel)
+{
+    if(_wl_handle){
+        log_w("Already Mounted!");
+        return true;
+    }     
+
+    if (!check_ffat_partition(partitionLabel)) return false;
+
+    esp_vfs_fat_mount_config_t conf = {
+      .format_if_mount_failed = formatOnFail,
+      .max_files = maxOpenFiles
+    };
+    esp_err_t err = esp_vfs_fat_spiflash_mount(basePath, partitionLabel, &conf, &_wl_handle);
+    if(err){
+        log_e("Mounting FFat partition failed! Error: %d", err);
+        return false;
+    }
+    _impl->mountpoint(basePath);
+    return true;
+}
+
+void F_Fat::end()
+{
+    if(_wl_handle){
+        esp_err_t err = esp_vfs_fat_spiflash_unmount(_impl->mountpoint(), _wl_handle);
+        if(err){
+            log_e("Unmounting FFat partition failed! Error: %d", err);
+            return;
+        }
+        _wl_handle = NULL;
+        _impl->mountpoint(NULL);
+    }
+}
+
+bool F_Fat::format(bool full_wipe, char* partitionLabel)
+{
+    esp_err_t result;
+    if(_wl_handle){
+        log_w("Already Mounted!");
+        return false;
+    }
+    wl_handle_t temp_handle;
+// Attempt to mount to see if there is already data
+    const esp_partition_t *ffat_partition = check_ffat_partition(partitionLabel);
+    if (!ffat_partition) return false;
+    result = wl_mount(ffat_partition, &temp_handle);
+
+    if (result == ESP_OK) {
+// Wipe disk- quick just wipes the FAT. Full zeroes the whole disk
+        uint32_t wipe_size = full_wipe ? wl_size(temp_handle) : 16384;
+        wl_erase_range(temp_handle, 0, wipe_size);
+        wl_unmount(temp_handle);
+    }
+// Now do a mount with format_if_fail (which it will)
+    esp_vfs_fat_mount_config_t conf = {
+      .format_if_mount_failed = true,
+      .max_files = 1
+    };
+    result = esp_vfs_fat_spiflash_mount("/format_ffat", partitionLabel, &conf, &temp_handle);
+    esp_vfs_fat_spiflash_unmount("/format_ffat", temp_handle);
+    return result;
+}
+
+size_t F_Fat::totalBytes()
+{
+    FATFS *fs;
+    DWORD free_clust, tot_sect, sect_size;
+
+    BYTE pdrv = ff_diskio_get_pdrv_wl(_wl_handle);
+    char drv[3] = {(char)(48+pdrv), ':', 0};
+    FRESULT res = f_getfree(drv, &free_clust, &fs);
+    tot_sect = (fs->n_fatent - 2) * fs->csize;
+    sect_size = CONFIG_WL_SECTOR_SIZE;
+    return tot_sect * sect_size;
+}
+
+size_t F_Fat::freeBytes()
+{
+
+    FATFS *fs;
+    DWORD free_clust, free_sect, sect_size;
+
+    BYTE pdrv = ff_diskio_get_pdrv_wl(_wl_handle);
+    char drv[3] = {(char)(48+pdrv), ':', 0};
+    FRESULT res = f_getfree(drv, &free_clust, &fs);
+    free_sect = free_clust * fs->csize;
+    sect_size = CONFIG_WL_SECTOR_SIZE;
+    return free_sect * sect_size;
+}
+
+bool F_Fat::exists(const char* path)
+{
+    File f = open(path, "r");
+    return (f == true) && !f.isDirectory();
+}
+
+bool F_Fat::exists(const String& path)
+{
+    return exists(path.c_str());
+}
+
+
+F_Fat FFat = F_Fat(FSImplPtr(new VFSImpl()));

libraries/FFat/src/FFat.h

@@ -0,0 +1,47 @@
+// Copyright 2015-2016 Espressif Systems (Shanghai) PTE LTD
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#ifndef _FFAT_H_
+#define _FFAT_H_
+
+#include "FS.h"
+#include "wear_levelling.h"
+
+#define FFAT_WIPE_QUICK 0
+#define FFAT_WIPE_FULL 1
+#define FFAT_PARTITION_LABEL "ffat"
+
+namespace fs
+{
+
+class F_Fat : public FS
+{
+public:
+    F_Fat(FSImplPtr impl);
+    bool begin(bool formatOnFail=false, const char * basePath="/ffat", uint8_t maxOpenFiles=10, const char * partitionLabel = (char*)FFAT_PARTITION_LABEL);
+    bool format(bool full_wipe = FFAT_WIPE_QUICK, char* partitionLabel = (char*)FFAT_PARTITION_LABEL);
+    size_t totalBytes();
+    size_t freeBytes();
+    void end();
+    bool exists(const char* path);
+    bool exists(const String& path);
+
+private:
+    wl_handle_t _wl_handle;
+};
+
+}
+
+extern fs::F_Fat FFat;
+
+#endif /* _FFAT_H_ */

libraries/WebServer/examples/FSBrowser/FSBrowser.ino

@@ -1,5 +1,5 @@
 /*
-  FSWebServer - Example WebServer with SPIFFS backend for esp8266
+  FSWebServer - Example WebServer with FS backend for esp8266/esp32
   Copyright (c) 2015 Hristo Gochkov. All rights reserved.
   This file is part of the WebServer library for Arduino environment.
 
@@ -26,14 +26,21 @@
 #include <WiFiClient.h>
 #include <WebServer.h>
 #include <ESPmDNS.h>
-#include <SPIFFS.h>
 
+#define FILESYSTEM SPIFFS
+#define FORMAT_FILESYSTEM true
 #define DBG_OUTPUT_PORT Serial
 
+#if FILESYSTEM == FFat
+#include <FFat.h>
+#endif
+#if FILESYSTEM == SPIFFS
+#include <SPIFFS.h>
+#endif
+
 const char* ssid = "wifi-ssid";
 const char* password = "wifi-password";
 const char* host = "esp32fs";
-
 WebServer server(80);
 //holds the current upload
 File fsUploadFile;
@@ -84,7 +91,7 @@ String getContentType(String filename) {
 
 bool exists(String path){
   bool yes = false;
-  File file = SPIFFS.open(path, "r");
+  File file = FILESYSTEM.open(path, "r");
   if(!file.isDirectory()){
     yes = true;
   }
@@ -103,7 +110,7 @@ bool handleFileRead(String path) {
     if (exists(pathWithGz)) {
       path += ".gz";
     }
-    File file = SPIFFS.open(path, "r");
+    File file = FILESYSTEM.open(path, "r");
     server.streamFile(file, contentType);
     file.close();
     return true;
@@ -122,7 +129,7 @@ void handleFileUpload() {
       filename = "/" + filename;
     }
     DBG_OUTPUT_PORT.print("handleFileUpload Name: "); DBG_OUTPUT_PORT.println(filename);
-    fsUploadFile = SPIFFS.open(filename, "w");
+    fsUploadFile = FILESYSTEM.open(filename, "w");
     filename = String();
   } else if (upload.status == UPLOAD_FILE_WRITE) {
     //DBG_OUTPUT_PORT.print("handleFileUpload Data: "); DBG_OUTPUT_PORT.println(upload.currentSize);
@@ -149,7 +156,7 @@ void handleFileDelete() {
   if (!exists(path)) {
     return server.send(404, "text/plain", "FileNotFound");
   }
-  SPIFFS.remove(path);
+  FILESYSTEM.remove(path);
   server.send(200, "text/plain", "");
   path = String();
 }
@@ -166,7 +173,7 @@ void handleFileCreate() {
   if (exists(path)) {
     return server.send(500, "text/plain", "FILE EXISTS");
   }
-  File file = SPIFFS.open(path, "w");
+  File file = FILESYSTEM.open(path, "w");
   if (file) {
     file.close();
   } else {
@@ -186,7 +193,7 @@ void handleFileList() {
   DBG_OUTPUT_PORT.println("handleFileList: " + path);
 
 
-  File root = SPIFFS.open(path);
+  File root = FILESYSTEM.open(path);
   path = String();
 
   String output = "[";
@@ -212,9 +219,10 @@ void setup(void) {
   DBG_OUTPUT_PORT.begin(115200);
   DBG_OUTPUT_PORT.print("\n");
   DBG_OUTPUT_PORT.setDebugOutput(true);
-  SPIFFS.begin();
+  if (FORMAT_FILESYSTEM) FILESYSTEM.format();
+  FILESYSTEM.begin();
   {
-      File root = SPIFFS.open("/");
+      File root = FILESYSTEM.open("/");
       File file = root.openNextFile();
       while(file){
           String fileName = file.name();
@@ -267,7 +275,7 @@ void setup(void) {
   }, handleFileUpload);
 
   //called when the url is not defined here
-  //use it to load content from SPIFFS
+  //use it to load content from FILESYSTEM
   server.onNotFound([]() {
     if (!handleFileRead(server.uri())) {
       server.send(404, "text/plain", "FileNotFound");

tools/partitions/ffat.csv

@@ -0,0 +1,7 @@
+# Name,   Type, SubType, Offset,  Size, Flags
+nvs,      data, nvs,     0x9000,  0x5000,
+otadata,  data, ota,     0xe000,  0x2000,
+app0,     app,  ota_0,   0x10000, 0x200000,
+app1,     app,  ota_1,   0x210000,0x200000,
+eeprom,   data, 0x99,    0x410000,0x1000,
+ffat,     data, fat,     0x411000,0xBEE000,