Index: src/edu/ucsb/nceas/metacat/MetacatHandler.java
===================================================================
--- src/edu/ucsb/nceas/metacat/MetacatHandler.java	(revision 6851)
+++ src/edu/ucsb/nceas/metacat/MetacatHandler.java	(revision 6852)
@@ -1849,7 +1849,7 @@
                   // handle inserts
                   try {
                 	// create the system metadata  
-                    sysMeta = SystemMetadataFactory.createSystemMetadata(newdocid, false);
+                    sysMeta = SystemMetadataFactory.createSystemMetadata(newdocid, false, false);
                     
                     // save it to the map
                     HazelcastService.getInstance().getSystemMetadataMap().put(sysMeta.getIdentifier(), sysMeta);
@@ -2992,7 +2992,7 @@
                                     username, groupnames);
                             
                             // generate system metadata about the doc
-                            SystemMetadata sm = SystemMetadataFactory.createSystemMetadata(docid, false);
+                            SystemMetadata sm = SystemMetadataFactory.createSystemMetadata(docid, false, false);
 							
 					        // manage it in the store
                             HazelcastService.getInstance().getSystemMetadataMap().put(sm.getIdentifier(), sm);
Index: src/edu/ucsb/nceas/metacat/dataone/SystemMetadataFactory.java
===================================================================
--- src/edu/ucsb/nceas/metacat/dataone/SystemMetadataFactory.java	(revision 6851)
+++ src/edu/ucsb/nceas/metacat/dataone/SystemMetadataFactory.java	(revision 6852)
@@ -25,10 +25,10 @@
  */
 package edu.ucsb.nceas.metacat.dataone;
 
-import java.io.BufferedInputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.math.BigInteger;
+import java.net.URL;
 import java.security.NoSuchAlgorithmException;
 import java.sql.SQLException;
 import java.util.ArrayList;
@@ -109,7 +109,7 @@
 	 * @throws AccessControlException 
 	 * @throws AccessException 
 	 */
-	public static SystemMetadata createSystemMetadata(String localId, boolean includeORE)
+	public static SystemMetadata createSystemMetadata(String localId, boolean includeORE, boolean downloadData)
 			throws McdbException, McdbDocNotFoundException, SQLException,
 			IOException, AccessionNumberException, ClassNotFoundException,
 			InsufficientKarmaException, ParseLSIDException,
@@ -340,18 +340,52 @@
 									.getFormat("application/octet-stream")
 									.getFormatId().getValue();
 						}
-						String dataDocLocalId = "";
+
+						// process the data
+						String dataDocLocalId = null;
+						Identifier dataGuid = new Identifier();
+
+						// handle ecogrid, or downloadable data
+						String ecogridPrefix = "ecogrid://knb/";
+						if (dataDocUrl.trim().startsWith(ecogridPrefix)) {
+							dataDocLocalId = dataDocUrl.substring(dataDocUrl.indexOf(ecogridPrefix) + ecogridPrefix.length());
+						} else {
+							// should we try downloading the remote data?
+							if (downloadData) {
+								InputStream dataObject = null;
+								try {
+									// download the data from the URL
+									URL dataURL = new URL(dataDocUrl);
+									dataObject = dataURL.openStream();
+									// TODO: check for valid content
+								} catch (Exception e) {
+									// error with the download
+									logMetacat.warn("Error downloading remote data. " + e.getMessage());
+								}
+								
+								if (dataObject != null) {
+									// create the local version of it
+									dataDocLocalId = DocumentUtil.generateDocumentId(1);
+									IdentifierManager.getInstance().createMapping(dataDocLocalId, dataDocLocalId);
+									dataGuid.setValue(dataDocLocalId);
+									
+									// save it locally
+									Session session = new Session();
+									session.setSubject(submitter);
+									MockHttpServletRequest request = new MockHttpServletRequest(null, null, null);
+									MNodeService.getInstance(request).insertDataObject(dataObject, dataGuid, session);
+								}
+							}
+							
+						}
+						
 						logMetacat.debug("Data local ID: " + dataDocLocalId);
 						logMetacat.debug("Data URL     : " + dataDocUrl);
 						logMetacat.debug("Data mime    : " + dataDocMimeType);
-	
-						// we only handle ecogrid urls right now
-						String ecogridPrefix = "ecogrid://knb/";
-						if (dataDocUrl.trim().startsWith(ecogridPrefix)) {
-							dataDocLocalId = dataDocUrl.substring(dataDocUrl
-									.indexOf(ecogridPrefix)
-									+ ecogridPrefix.length());
-	
+						
+						// now we have a local id for the data
+						if (dataDocLocalId != null) {
+							
 							// look up the guid for the data
 							String dataDocid = DocumentUtil.getSmartDocId(dataDocLocalId);
 							int dataRev = DocumentUtil.getRevisionFromAccessionNumber(dataDocLocalId);
@@ -359,7 +393,6 @@
 							// check if data system metadata exists already
 							SystemMetadata dataSysMeta = null;
 							String dataGuidString = null;
-							Identifier dataGuid = new Identifier();
 							try {
 								// look for the identifier
 								dataGuidString = IdentifierManager.getInstance().getGUID(dataDocid, dataRev);
@@ -384,7 +417,7 @@
 								logMetacat.debug("There was not an existing system metadata document for " + dataDocLocalId);
 								try {
 									logMetacat.debug("Creating a system metadata " + "document for " + dataDocLocalId);
-									dataSysMeta = createSystemMetadata(dataDocLocalId, includeORE);
+									dataSysMeta = createSystemMetadata(dataDocLocalId, includeORE, false);
 	
 									// now look it up again
 									dataGuidString = IdentifierManager.getInstance().getGUID(dataDocid, dataRev);
Index: src/edu/ucsb/nceas/metacat/admin/upgrade/GenerateSystemMetadata.java
===================================================================
--- src/edu/ucsb/nceas/metacat/admin/upgrade/GenerateSystemMetadata.java	(revision 6851)
+++ src/edu/ucsb/nceas/metacat/admin/upgrade/GenerateSystemMetadata.java	(revision 6852)
@@ -61,8 +61,10 @@
         boolean success = true;
         // include ORE?
         boolean includeOre = true;
+        boolean downloadData = true;
+
         try {
-			generateMissingSystemMetadata(includeOre);
+			generateMissingSystemMetadata(includeOre, downloadData);
 		} catch (Exception e) {
 			String msg = "Problem generating missing system metadata: " + e.getMessage();
 			log.error(msg, e);
@@ -95,7 +97,7 @@
      * @throws McdbDocNotFoundException 
      * @throws ServiceFailure 
      */
-    public void generateMissingSystemMetadata(boolean includeOre) 
+    public void generateMissingSystemMetadata(boolean includeOre, boolean downloadData) 
     throws ServiceFailure, McdbDocNotFoundException, PropertyNotFoundException, InvalidToken, NotAuthorized, 
     NotFound, NotImplemented, InvalidRequest, NoSuchAlgorithmException, AccessionNumberException, SQLException 
     {
@@ -107,7 +109,7 @@
             //for each id, add a system metadata doc
         	try {
         		log.debug("generating missing system metadata for " + localId);
-        		generateMissingSystemMetadata(localId, includeOre);
+        		generateMissingSystemMetadata(localId, includeOre, downloadData);
         	} catch (Exception e) {
 				log.error("Error generating system metadata for: " + localId, e);
 			}
@@ -137,7 +139,7 @@
      * @throws PropertyNotFoundException 
      * @throws McdbDocNotFoundException 
      */
-    public void generateMissingSystemMetadata(String localId, boolean includeOre) 
+    public void generateMissingSystemMetadata(String localId, boolean includeOre, boolean downloadData) 
     throws ServiceFailure, McdbDocNotFoundException, PropertyNotFoundException, InvalidToken, NotAuthorized,
     NotFound, NotImplemented, InvalidRequest, NoSuchAlgorithmException, AccessionNumberException, SQLException 
     {
@@ -147,7 +149,7 @@
 
         //generate required system metadata fields from the document
         try {
-        	sm = SystemMetadataFactory.createSystemMetadata(localId, includeOre);
+        	sm = SystemMetadataFactory.createSystemMetadata(localId, includeOre, downloadData);
         } catch (Exception e1) {
         	e1.printStackTrace();
         	ServiceFailure sf = new ServiceFailure("00","Exception in generateMissingSystemMetadata: " +
