This is the mail archive of the cygwin-developers mailing list for the Cygwin project.


Index Nav: [Date Index] [Subject Index] [Author Index] [Thread Index]
Message Nav: [Date Prev] [Date Next] [Thread Prev] [Thread Next]
Other format: [Raw text]

Re: Cygwin Filesystem Performance degradation 1.7.5 vs 1.7.7, and methods for improving performance


Hi,

> I will send multiple patches later today
patch for caching symlink content + exec_state.

Yoni.

Index: fhandler_disk_file.cc
===================================================================
RCS file: /cvs/src/src/winsup/cygwin/fhandler_disk_file.cc,v
retrieving revision 1.345
diff -u -p -r1.345 fhandler_disk_file.cc
--- fhandler_disk_file.cc	7 Oct 2010 14:03:26 -0000	1.345
+++ fhandler_disk_file.cc	13 Oct 2010 15:29:36 -0000
@@ -348,7 +348,7 @@ fhandler_base::fstat_by_handle (struct _
      on the information stored in pc.fnoi.  So we overwrite them here. */
   if (get_io_handle ())
     {
-      PFILE_NETWORK_OPEN_INFORMATION pfnoi = pc.fnoi ();
+      PFILE_CYGWIN_INFORMATION pfnoi = pc.fnoi ();
       status = NtQueryInformationFile (h, &io, pfnoi, sizeof *pfnoi,
                                       FileNetworkOpenInformation);
       if (!NT_SUCCESS (status))
@@ -485,7 +485,7 @@ fhandler_base::fstat_helper (struct __st
   IO_STATUS_BLOCK st;
   FILE_COMPRESSION_INFORMATION fci;
   HANDLE h = get_stat_handle ();
-  PFILE_NETWORK_OPEN_INFORMATION pfnoi = pc.fnoi ();
+  PFILE_CYGWIN_INFORMATION pfnoi = pc.fnoi ();
   ULONG attributes = pc.file_attributes ();

   to_timestruc_t ((PFILETIME) &pfnoi->LastAccessTime, &buf->st_atim);
@@ -590,8 +590,14 @@ fhandler_base::fstat_helper (struct __st
       else
 	{
 	  buf->st_mode |= S_IFREG;
+	  file_cache_t *s = NULL;
+	  if ((s = file_cache_lookup(pc.get_win32(), pfnoi)) &&
+	      s->exec_state>=0)
+	    {
+	      pc.set_exec(s->exec_state);
+	    }
 	  /* Check suffix for executable file. */
-	  if (pc.exec_state () != is_executable)
+	  else if (pc.exec_state () != is_executable)
 	    {
 	      PUNICODE_STRING path = pc.get_nt_native_path ();

@@ -638,6 +644,11 @@ fhandler_base::fstat_helper (struct __st
 		  NtClose (h);
 		}
 	    }
+	    if (!s)
+	    {
+		file_cache_update(pc.get_win32(), pfnoi, NULL, 0,
+		    pc.exec_state());
+	    }
 	}
       if (pc.exec_state () == is_executable)
 	buf->st_mode |= STD_XBITS;
Index: path.cc
===================================================================
RCS file: /cvs/src/src/winsup/cygwin/path.cc,v
retrieving revision 1.617
diff -u -p -r1.617 path.cc
--- path.cc	9 Oct 2010 10:54:13 -0000	1.617
+++ path.cc	13 Oct 2010 15:29:39 -0000
@@ -73,6 +73,7 @@
 #include <wctype.h>

 bool dos_file_warning = true;
+bool use_file_cache = true;

 suffix_info stat_suffixes[] =
 {
@@ -1876,6 +1877,7 @@ symlink_info::check_sysfile (HANDLE h)
       else
 	res = posixify (srcbuf);
     }
+  debug_printf ("res %d", res);
   return res;
 }

@@ -2213,6 +2215,95 @@ symlink_info::parse_device (const char *
    Return -1 on error, 0 if PATH is not a symlink, or the length
    stored into BUF if PATH is a symlink.  */

+static inline unsigned int hash_from_int(unsigned int val)
+{
+    return val * 1103515245 + 12345; /* from rand() implementaion */
+}
+
+static inline unsigned int hash_from_mem(const char *buf, int len)
+{
+    unsigned int val = 0;
+    int i = 0, len4 = len & ~3;
+    for (; i<len4; i+=4)
+        val = hash_from_int(val ^ *(int *)(buf+i));
+    for (; i<len; i++)
+        val = hash_from_int(val ^ buf[i]);
+    return val;
+}
+
+static inline unsigned int hash_from_str(const char *str)
+{
+    return hash_from_mem(str, strlen(str));
+}
+
+#define FILE_CACHE_SIZE 1000
+#define FILE_CACHE_BUCKET 10
+typedef struct {
+    file_cache_t *f[FILE_CACHE_BUCKET];
+    int i;
+} file_cache_hash_t;
+static file_cache_hash_t file_cache[FILE_CACHE_SIZE];
+
+void file_cache_update(const char *path,
+    FILE_CYGWIN_INFORMATION *fnoi, char *contents, int symlink_len,
+    int exec_state)
+{
+    if (!use_file_cache)
+	return;
+    unsigned int hash = hash_from_str(path), bucket = hash%FILE_CACHE_SIZE;
+    int i;
+    file_cache_hash_t *c = &file_cache[bucket];
+    file_cache_t *s = (file_cache_t*)calloc(sizeof(file_cache_t), 1), *_s;
+    s->path = strdup(path);
+    s->hash = hash;
+    if (exec_state>=0)
+	s->exec_state = exec_state;
+    if (contents)
+	s->contents = strdup(contents);
+    if (symlink_len)
+	s->symlink_len = symlink_len;
+    memcpy(&s->fnoi, fnoi, sizeof(FILE_CYGWIN_INFORMATION));
+    /* if the same file is already in the cache - replace it */
+    for (i=0; i<FILE_CACHE_BUCKET && c->f[i] && c->f[i]->hash!=hash; i++);
+    if (i==FILE_CACHE_BUCKET)
+    {
+	/* file not yet in cache - do round robin */
+	i = c->i;
+	c->i = (c->i+1) % FILE_CACHE_BUCKET;
+    }
+    if ((_s = c->f[i]))
+    {
+	free(_s->path);
+	free(_s);
+    }
+    c->f[i] = s;
+}
+
+file_cache_t *file_cache_lookup(const char *path,
+    FILE_CYGWIN_INFORMATION *fnoi)
+{
+    if (!use_file_cache)
+	return NULL;
+    unsigned int hash = hash_from_str(path), bucket = hash%FILE_CACHE_SIZE;
+    file_cache_hash_t *c = &file_cache[bucket];
+    file_cache_t *s = NULL;
+    for (int i=0; i<FILE_CACHE_BUCKET; i++)
+    {
+	s = c->f[i];
+	if (!s)
+	    return NULL;
+	if (s->hash != hash || strcmp(path, s->path))
+	    continue;
+	/* its the same file */
+	return fnoi->CreationTime.QuadPart==s->fnoi.CreationTime.QuadPart &&
+	    fnoi->LastWriteTime.QuadPart==s->fnoi.LastWriteTime.QuadPart &&
+	    fnoi->ChangeTime.QuadPart==s->fnoi.ChangeTime.QuadPart &&
+	    fnoi->AllocationSize.QuadPart==s->fnoi.AllocationSize.QuadPart &&
+	    fnoi->FileAttributes==s->fnoi.FileAttributes ? s : NULL;
+    }
+    return NULL;
+}
+
 int
 symlink_info::check (char *path, const suffix_info *suffixes, fs_info &fs,
 		     path_conv_handle &conv_hdl)
@@ -2386,7 +2477,7 @@ restart:
 	    }
 	  else
 	    {
-	      PFILE_NETWORK_OPEN_INFORMATION pfnoi = conv_hdl.fnoi ();
+	      PFILE_CYGWIN_INFORMATION pfnoi = conv_hdl.fnoi ();

 	      status = NtQueryInformationFile (h, &io, pfnoi, sizeof *pfnoi,
 					       FileNetworkOpenInformation);
@@ -2500,7 +2591,7 @@ restart:
 		    }
 		  else
 		    {
-		      PFILE_NETWORK_OPEN_INFORMATION pfnoi = conv_hdl.fnoi ();
+		      PFILE_CYGWIN_INFORMATION pfnoi = conv_hdl.fnoi ();

 		      fileattr = fdi_buf.fdi.FileAttributes;
 		      memcpy (pfnoi, &fdi_buf.fdi.CreationTime, sizeof *pfnoi);
@@ -2622,18 +2713,28 @@ restart:
 	       == FILE_ATTRIBUTE_SYSTEM)
 	{
 	  HANDLE sym_h;
-
-	  status = NtOpenFile (&sym_h, SYNCHRONIZE | GENERIC_READ, &attr, &io,
-			       FILE_SHARE_VALID_FLAGS,
-			       FILE_OPEN_FOR_BACKUP_INTENT
-			       | FILE_SYNCHRONOUS_IO_NONALERT);
-
-	  if (!NT_SUCCESS (status))
-	    res = 0;
+	  file_cache_t *s = NULL;
+	  if ((s = file_cache_lookup(path, conv_hdl.fnoi ())) &&
+	      (res = s->symlink_len)>0)
+	    {
+	      strcpy(contents, s->contents);
+	      pflags |= PATH_SYMLINK;
+	    }
 	  else
 	    {
-	      res = check_sysfile (sym_h);
-	      NtClose (sym_h);
+	      status = NtOpenFile (&sym_h, SYNCHRONIZE | GENERIC_READ, &attr,
+		                   &io, FILE_SHARE_VALID_FLAGS,
+			           FILE_OPEN_FOR_BACKUP_INTENT
+			           | FILE_SYNCHRONOUS_IO_NONALERT);
+
+	      if (!NT_SUCCESS (status))
+	        res = 0;
+	      else
+	        {
+	          res = check_sysfile (sym_h);
+		  file_cache_update(path, conv_hdl.fnoi (), contents, res, -1);
+	          NtClose (sym_h);
+	        }
 	    }
 	  if (res)
 	    break;
Index: path.h
===================================================================
RCS file: /cvs/src/src/winsup/cygwin/path.h,v
retrieving revision 1.152
diff -u -p -r1.152 path.h
--- path.h	7 Oct 2010 14:03:26 -0000	1.152
+++ path.h	13 Oct 2010 15:29:41 -0000
@@ -90,23 +90,26 @@ enum path_types
 };

 class symlink_info;
-struct _FILE_NETWORK_OPEN_INFORMATION;
+/* Identical to FILE_NETWORK_OPEN_INFORMATION.  We don't want to pull in
+   ntdll.h here, though. */
+typedef struct _FILE_CYGWIN_INFORMATION
+{
+  LARGE_INTEGER CreationTime;
+  LARGE_INTEGER LastAccessTime;
+  LARGE_INTEGER LastWriteTime;
+  LARGE_INTEGER ChangeTime;
+  LARGE_INTEGER AllocationSize;
+  LARGE_INTEGER EndOfFile;
+  ULONG FileAttributes;
+  ULONG ReparseTag;
+  LARGE_INTEGER FileId;
+} FILE_CYGWIN_INFORMATION, *PFILE_CYGWIN_INFORMATION;

class path_conv_handle
{
HANDLE hdl;
union {
- /* Identical to FILE_NETWORK_OPEN_INFORMATION. We don't want to pull in
- ntdll.h here, though. */
- struct {
- LARGE_INTEGER CreationTime;
- LARGE_INTEGER LastAccessTime;
- LARGE_INTEGER LastWriteTime;
- LARGE_INTEGER ChangeTime;
- LARGE_INTEGER AllocationSize;
- LARGE_INTEGER EndOfFile;
- ULONG FileAttributes;
- } _fnoi;
+ FILE_CYGWIN_INFORMATION _fnoi;
/* For NFS. */
fattr3 _fattr3;
} attribs;
@@ -127,12 +130,26 @@ public:
hdl = NULL;
}
inline HANDLE handle () const { return hdl; }
- inline struct _FILE_NETWORK_OPEN_INFORMATION *fnoi ()
- { return (struct _FILE_NETWORK_OPEN_INFORMATION *) &attribs._fnoi; }
+ inline FILE_CYGWIN_INFORMATION *fnoi ()
+ { return (FILE_CYGWIN_INFORMATION *) &attribs._fnoi; }
inline struct fattr3 *nfsattr ()
{ return (struct fattr3 *) &attribs._fattr3; }
};


+typedef struct {
+ FILE_CYGWIN_INFORMATION fnoi;
+ char *path;
+ char *contents;
+ int symlink_len;
+ int exec_state;
+ unsigned int hash;
+} file_cache_t;
+
+void file_cache_update(const char *path,
+ FILE_CYGWIN_INFORMATION *fnoi, char *contents, int symlink_len,
+ int exec_state);
+file_cache_t *file_cache_lookup(const char *path, FILE_CYGWIN_INFORMATION *fnoi);
+
class path_conv
{
DWORD fileattr;
@@ -320,7 +337,7 @@ class path_conv
bool is_binary ();


HANDLE handle () const { return conv_handle.handle (); }
- struct _FILE_NETWORK_OPEN_INFORMATION *fnoi () { return conv_handle.fnoi (); }
+ FILE_CYGWIN_INFORMATION *fnoi () { return conv_handle.fnoi (); }
struct fattr3 *nfsattr () { return conv_handle.nfsattr (); }
void reset_conv_handle () { conv_handle.set (NULL); }
void close_conv_handle () { conv_handle.close (); }



Index Nav: [Date Index] [Subject Index] [Author Index] [Thread Index]
Message Nav: [Date Prev] [Date Next] [Thread Prev] [Thread Next]