Index: DarwinCoreDataSource.java =================================================================== RCS file: /cvs/kepler/src/org/ecoinformatics/seek/datasource/darwincore/DarwinCoreDataSource.java,v retrieving revision 1.23 diff -u -r1.23 DarwinCoreDataSource.java --- DarwinCoreDataSource.java 19 Jul 2005 17:14:20 -0000 1.23 +++ DarwinCoreDataSource.java 13 Sep 2005 15:45:20 -0000 @@ -31,7 +31,6 @@ package org.ecoinformatics.seek.datasource.darwincore; -import java.io.File; import java.io.StringReader; import java.sql.ResultSet; import java.sql.ResultSetMetaData; @@ -39,7 +38,6 @@ import java.util.Enumeration; import java.util.Hashtable; import java.util.List; -import java.util.StringTokenizer; import java.util.Vector; import org.ecoinformatics.ecogrid.EcogridQueryParser; @@ -47,10 +45,6 @@ import org.ecoinformatics.ecogrid.ResultsetType; import org.ecoinformatics.ecogrid.ResultsetType_record; import org.ecoinformatics.ecogrid.ResultsetType_record_returnField; -import org.ecoinformatics.ecogrid.digir.impl.DigirJavaToEcogridJavaResultsetTransformer; -import org.ecoinformatics.ecogrid.digir.impl.DigirProviderInfo; -import org.ecoinformatics.ecogrid.digir.impl.DigirProxyImpl; -import org.ecoinformatics.ecogrid.digir.impl.EcogridJavaToDigirJavaQueryTransformer; import org.ecoinformatics.seek.dataquery.DBTableNameResolver; import org.ecoinformatics.seek.dataquery.DBTablesGenerator; import org.ecoinformatics.seek.dataquery.HsqlDataQueryAction; @@ -72,6 +66,7 @@ import org.ecoinformatics.seek.querybuilder.DBQueryDef; import org.ecoinformatics.seek.querybuilder.DBQueryDefParserEmitter; import org.ecoinformatics.seek.querybuilder.DBSchemaParserEmitter; +import org.ecoinformatics.seek.datasource.EcogridQueryDataCacheItem; import org.ecoinformatics.seek.datasource.EcogridDataCacheItem; import ptolemy.actor.TypedIOPort; @@ -116,7 +111,7 @@ /** Output indicator parameter. */ - private int _outputType = _NOPORTTYPE; + private int _outputType = _ASFIELD; private Entity _tableEntity = null; private int _recordCount = 0; @@ -130,10 +125,10 @@ private boolean _schemaAttrIsSet = false; private DBTablesGenerator _tableGenerator = null; - private EcogridDataCacheItem _cachedDataItem = null; + private EcogridQueryDataCacheItem _cachedDataItem = null; private EcogridDataCacheItem _tableDataCache = null; - private ResultsetType _resultsetType = null; - private ResultsetType_record[] _records = null; + + private Vector[] _dataVectors = null; /** Icon indicating the communication region. */ @@ -146,7 +141,6 @@ private static DSSchemaIFace _darwinCoreSchema = null; // Constants used for more efficient execution. - private static final int _NOPORTTYPE = 0; private static final int _ASTABLE = 1; private static final int _ASROW = 2; private static final int _ASFIELD = 3; @@ -215,7 +209,7 @@ _outputTypeAttr.addChoice("As Field"); _outputTypeAttr.addChoice("As Table"); _outputTypeAttr.addChoice("As Row"); - _outputType = _NOPORTTYPE; + _outputType = _ASFIELD; _schemaAttr = new StringAttribute(this, "schemaDef"); TextStyle schemaDefTS = new TextStyle(_schemaAttr, "schemaDef"); @@ -377,10 +371,13 @@ * * @return */ - public String getInitFromData() + public boolean isInitFromData() { - - return getStrAttr(_initFromData, INITFROMDATA_ATTR); + String initfromdata = getStrAttr( _initFromData, INITFROMDATA_ATTR ); + if ( initfromdata == null ) { + return false; + } + return ( initfromdata.equals( "yes" )); } /** * Set the endpoint of this record. The endpoint indicates where the service @@ -502,34 +499,30 @@ */ private Vector[] transformResultSet(ResultSet aResultset) throws SQLException { - Vector[] dataArray = null; - if (aResultset != null) + if (aResultset == null) { - Vector data = new Vector(); - ResultSetMetaData metadata = aResultset.getMetaData(); - if (metadata != null) - { - int columnSize = metadata.getColumnCount(); - - while (aResultset.next()) { - Vector rowData = new Vector(); - for (int i = 0; i < columnSize; i++) { - String str = aResultset.getString(i + 1); - rowData.add(str); - } - data.add(rowData); - } - dataArray = transformVectorToArray(data); - } - else - { - System.err.println("Metadata is NULL for resultset."); - } + System.err.println("Resultset is NULL in transformResultSet."); + return null; } - else + ResultSetMetaData metadata = aResultset.getMetaData(); + if (metadata == null) { - System.err.println("Resultset is NULL in transformResultSet."); + System.err.println("Metadata is NULL for resultset."); + return null; + } + int columnSize = metadata.getColumnCount(); + + Vector data = new Vector(); + while (aResultset.next()) { + Vector rowData = new Vector(); + for (int i = 0; i < columnSize; i++) { + String str = aResultset.getString(i + 1); + rowData.add(str); + } + data.add(rowData); } + Vector[] dataArray = null; + dataArray = transformVectorToArray(data); return dataArray; } @@ -642,7 +635,7 @@ * @param aRS * @return */ - private String createTableFromResultset(boolean aIncludeHeader, String aDelim) + private String createTableFromResultset(boolean aIncludeHeader, String aDelim, ResultsetType_record[] records) { StringBuffer tableStr = new StringBuffer(); StringBuffer rowOfData = new StringBuffer(); @@ -655,7 +648,7 @@ appendHeaderRow(tableStr, columns, delim, ROWDELIM); } - for (int i=0;i<_records.length;i++) + for (int i=0;i 0) { @@ -943,7 +936,6 @@ */ public void addPort(NamedObj aContainer, String aName, String aType) { - NamedObj composite = (NamedObj) aContainer.getContainer(); StringBuffer moml = new StringBuffer(); moml.append("\n\n"); moml.append("\n"); @@ -1196,174 +1188,123 @@ } /** - * Helper method to convert a comma separated list of providers into - * a Vector of DigirProviderInfo objects - * @param aProviderStr the list as a strng separated by comma's - * @return the vector of the list - */ - private Vector getProviderList(String aProviderStr) - { - if (aProviderStr.trim().length() == 0) return null; - - Vector list = new Vector(); - - StringTokenizer st = new StringTokenizer(aProviderStr, ","); - while (st.hasMoreTokens()) { - String providerStr = (String)st.nextToken(); - int inx = providerStr.indexOf('?'); - String baseURL = providerStr.substring(0, inx); - inx = providerStr.lastIndexOf('='); - String resource = providerStr.substring(inx+1, providerStr.length()).replaceAll("%20", " "); - - list.addElement(new DigirProviderInfo(baseURL, resource)); - } - return list; - } - - /** * Callback for changes in attribute values. */ public void attributeChanged(ptolemy.kernel.util.Attribute attribute) - throws ptolemy.kernel.util.IllegalActionException + throws ptolemy.kernel.util.IllegalActionException { - //System.out.println("Changed: "+attribute.getName()+" "+((Settable)attribute).getExpression()); - if (attribute == _sqlAttr) { - String sqlDef = ((Settable)attribute).getDefaultExpression(); - sqlDef = ((Settable)attribute).getExpression(); - if (sqlDef.length() > 0) - { - _queryDef = DBQueryDefParserEmitter.parseQueryDef(_schemaDef, sqlDef); - _columns = _queryDef.getSelects(); - if (_outputType != _NOPORTTYPE) - { - reconfigurePorts(_queryDef.getSelects(), _colTypes); - } - } - - } else if (attribute == _schemaAttr && !_ignoreSchemaChange) // NOTE: We may skip setting it here because _ignoreSchemaChange may be true - { - String schemaDef = ((Settable)_schemaAttr).getExpression(); - - // MOML may have a blank definition - if (schemaDef.length() > 0) - { - _schemaAttrIsSet = true; // remember that we have been set by the MOML - - dbg.print("schemaDef >>"+schemaDef+"<<", 2); - - _schemaDef = DBSchemaParserEmitter.parseSchemaDef(schemaDef); - } - - } else if (attribute == _outputTypeAttr) { - String strOutputType = _outputTypeAttr.stringValue(); - if (strOutputType.equals("As Table") && _outputType != _ASTABLE) { - _outputType = _ASTABLE; - reconfigurePortsAsTable(); - - } else if (strOutputType.equals("As Row") && _outputType != _ASROW) { - _outputType = _ASROW; - reconfigurePortsAsRow(); - - } else if (strOutputType.equals("As Field") && _outputType != _ASFIELD) { - _outputType = _ASFIELD; - if (_columns != null && _columns.size() > 0 && - _colTypes != null && _colTypes.size() > 0) { - reconfigurePortsAsField(_columns, _colTypes); - } - } else { - //throw new IllegalActionException(this, - // "Unrecognized outputType: " + strOutputType); + //System.out.println("Changed: "+attribute.getName()+" "+((Settable)attribute).getExpression()); + if (attribute == _sqlAttr) { + String sqlDef = ((Settable)attribute).getDefaultExpression(); + sqlDef = ((Settable)attribute).getExpression(); + if (sqlDef.length() > 0) + { + _queryDef = DBQueryDefParserEmitter.parseQueryDef(_schemaDef, sqlDef); + _columns = _queryDef.getSelects(); + reconfigurePorts(_queryDef.getSelects(), _colTypes); + } + + } else if (attribute == _schemaAttr && !_ignoreSchemaChange) // NOTE: We may skip setting it here because _ignoreSchemaChange may be true + { + String schemaDef = ((Settable)_schemaAttr).getExpression(); + + // MOML may have a blank definition + if (schemaDef.length() > 0) + { + _schemaAttrIsSet = true; // remember that we have been set by the MOML + + dbg.print("schemaDef >>"+schemaDef+"<<", 2); + + _schemaDef = DBSchemaParserEmitter.parseSchemaDef(schemaDef); + } + + } else if (attribute == _outputTypeAttr) { + String strOutputType = _outputTypeAttr.stringValue(); + if (strOutputType.equals("As Table") && _outputType != _ASTABLE) { + _outputType = _ASTABLE; + reconfigurePortsAsTable(); + + } else if (strOutputType.equals("As Row") && _outputType != _ASROW) { + _outputType = _ASROW; + reconfigurePortsAsRow(); + + } else if (strOutputType.equals("As Field") && _outputType != _ASFIELD) { + _outputType = _ASFIELD; + if (_columns != null && _columns.size() > 0 && + _colTypes != null && _colTypes.size() > 0) { + reconfigurePortsAsField(_columns, _colTypes); + } + } else { + //throw new IllegalActionException(this, + // "Unrecognized outputType: " + strOutputType); + } + + + } else if (attribute.getName().equals(PROVIDERS_ATTR) && + !(attribute.getContainer().getContainer() instanceof ResultTreeRoot)) + { + setIconStatus(TITLE_BUSY, RED); + + String endPointStr = getEndpoint(); + String searchStr = getSearchData(); + + //System.out.println("endPointStr: "+endPointStr); + //System.out.println("searchStr: "+searchStr); + + StringBuffer queryBuf = new StringBuffer(); + queryBuf.append(""); + queryBuf.append(""); + queryBuf.append(" http://digir.net/schema/conceptual/darwin/2003/1.0"); + queryBuf.append(" " + searchStr +" Query"); + + boolean useAllField = false; + if (useAllField) + { + DSTableIFace table = (DSTableIFace)_darwinCoreSchema.getTables().elementAt(0); // there is only one table + for (Enumeration e = table.getFields().elements(); e.hasMoreElements();) + { + DSTableFieldIFace field = (DSTableFieldIFace)e.nextElement(); + queryBuf.append(" /" + field.getName() + ""); + } + } else + { + queryBuf.append(" /Species"); + queryBuf.append(" /ScientificName"); + queryBuf.append(" /Collector"); + queryBuf.append(" /YearCollected"); + queryBuf.append(" /InstitutionCode"); + queryBuf.append(" /CollectionCode"); + queryBuf.append(" /CatalogNumber"); + queryBuf.append(" /CatalogNumberText"); + queryBuf.append(" /DecimalLatitude"); + queryBuf.append(" /DecimalLongitude"); + } + queryBuf.append(" " + searchStr +""); + queryBuf.append(""); + + try + { + QueryType query = null; + + StringReader strReader = new StringReader(queryBuf.toString()); + EcogridQueryParser parser = new EcogridQueryParser(strReader); + parser.parseXML(); + query = parser.getEcogridQuery(); + + setIconStatus(TITLE_BUSY, RED); + String resName = "DigirQuery: " + searchStr; + _cachedDataItem = (EcogridQueryDataCacheItem)DataCacheManager.getCacheItem(this, resName, endPointStr, EcogridQueryDataCacheItem.class.getName(), true); + _cachedDataItem.setQuery(query); + _cachedDataItem.start(); + } catch (Exception e ) + { + System.err.println(e); + e.printStackTrace(); + } } - - - } else if (attribute.getName().equals(PROVIDERS_ATTR) && - !(attribute.getContainer().getContainer() instanceof ResultTreeRoot)) - { - setIconStatus(TITLE_BUSY, RED); - - String endPointStr = getEndpoint(); - String providerStr = getProvider(); - String searchStr = getSearchData(); - - //System.out.println("providerStr: "+providerStr); - //System.out.println("endPointStr: "+endPointStr); - //System.out.println("searchStr: "+searchStr); - - Vector providerList = getProviderList(providerStr); - - StringBuffer queryBuf = new StringBuffer(); - queryBuf.append(""); - queryBuf.append(""); - queryBuf.append(" http://digir.net/schema/conceptual/darwin/2003/1.0"); - queryBuf.append(" " + searchStr +" Query"); - - boolean useAllField = false; - if (useAllField) - { - DSTableIFace table = (DSTableIFace)_darwinCoreSchema.getTables().elementAt(0); // there is only one table - for (Enumeration e = table.getFields().elements(); e.hasMoreElements();) - { - DSTableFieldIFace field = (DSTableFieldIFace)e.nextElement(); - queryBuf.append(" /" + field.getName() + ""); - } - } else - { - queryBuf.append(" /Species"); - queryBuf.append(" /ScientificName"); - queryBuf.append(" /Collector"); - queryBuf.append(" /YearCollected"); - queryBuf.append(" /InstitutionCode"); - queryBuf.append(" /CollectionCode"); - queryBuf.append(" /CatalogNumber"); - queryBuf.append(" /CatalogNumberText"); - queryBuf.append(" /DecimalLatitude"); - queryBuf.append(" /DecimalLongitude"); - } - queryBuf.append(" " + searchStr +""); - queryBuf.append(""); - - if (providerList.size() > 0) - { - try - { - QueryType query = null; - - // XXX no error checking - int inx = providerStr.indexOf('?'); - String baseURL = providerStr.substring(0, inx); - inx = providerStr.lastIndexOf('='); - String resource = providerStr.substring(inx+1, providerStr.length()).replaceAll("%20", " "); - //System.out.println(queryBuf.toString()); - - StringReader strReader = new StringReader(queryBuf.toString()); - EcogridQueryParser parser = new EcogridQueryParser(strReader); - parser.parseXML(); - query = parser.getEcogridQuery(); - - EcogridJavaToDigirJavaQueryTransformer transformer = new EcogridJavaToDigirJavaQueryTransformer(); - String digirQueryStr = transformer.transform(query, providerList); - //System.out.println(digirQueryStr); - if (digirQueryStr.length() > 0) - { - setIconStatus(TITLE_BUSY, RED); - String resName = "ProviderData " + providerStr; - String fullURL = DigirProxyImpl.getFullDigirURL(digirQueryStr); - _cachedDataItem = (EcogridDataCacheItem)DataCacheManager.getCacheItem(this, resName, fullURL, EcogridDataCacheItem.class.getName(), true); - _cachedDataItem.start(); - } - } catch (Exception e ) - { - System.err.println(e); - e.printStackTrace(); - } - } else - { - setIconStatus(TITLE_ERROR, MAGENTA); - } - } } /** @@ -1411,7 +1352,7 @@ } - private void generateTable() + private void generateTable( ResultsetType_record[] records ) { setIconStatus(TITLE_BUSY, RED); try @@ -1430,7 +1371,7 @@ // Ok, now create the table as a "string" table where each row is ROWDELIM // and load it into the cache. _tableDataCache = (EcogridDataCacheItem)DataCacheManager.getCacheItem(this, tableName, DATATABLE, EcogridDataCacheItem.class.getName(), false); - String tableStr = createTableFromResultset(true, "\t"); + String tableStr = createTableFromResultset(true, "\t", records ); //System.err.println("*** Table[\n"+tableStr+"\n]"); _tableDataCache.setData(tableStr.getBytes()); _tableDataCache.setStatus(DataCacheObject.CACHE_COMPLETE); @@ -1439,7 +1380,7 @@ // now create the table entity _tableEntity = new Entity(_tableDataCache.getName(), tableName, "", new Boolean(true), - Entity.COLUMNMAJOR, _records.length); + Entity.COLUMNMAJOR, records.length); _tableEntity.setNumHeaderLines(1); _tableEntity.setDelimiter("\\t"); _tableEntity.setRecordDelimiter(ROWDELIM); @@ -1488,122 +1429,85 @@ public void complete(DataCacheObject aItem) { - setIconStatus(TITLE_BINARY, YELLOW); - if (_cachedDataItem.isReady()) + if (!_cachedDataItem.isReady()) { - - boolean resultsetWasOK = false; - try - { - // this code is for thread problem, sometimes the method reconfigurePorts - // was called before getting _outputType - if (_outputType == _NOPORTTYPE) + setIconStatus(TITLE_ERROR, MAGENTA); + return; + } + + try { + // this code is for thread problem, sometimes the method reconfigurePorts + // was called before getting _outputType + String strOutputType = _outputTypeAttr.stringValue(); + if (strOutputType.equals("As Table")) { - String strOutputType = _outputTypeAttr.stringValue(); - if (strOutputType.equals("As Table")) - { - _outputType = _ASTABLE; - } - else if (strOutputType.equals("As Row")) - { - _outputType = _ASROW; - } - else if (strOutputType.equals("As Field")) - { - _outputType = _ASFIELD; - } - - } - String cachedFileName = _cachedDataItem.getLocalFileName(); - if (cachedFileName != null && cachedFileName.length() > 0) + _outputType = _ASTABLE; + } + else if (strOutputType.equals("As Row")) { - File cachedDataFile = new File(cachedFileName); - if (cachedDataFile.length() > 0) - { - DigirJavaToEcogridJavaResultsetTransformer ecogrid2ResultTransformer = new DigirJavaToEcogridJavaResultsetTransformer(); - _resultsetType = ecogrid2ResultTransformer.transform(cachedDataFile); - if (_resultsetType != null) - { - //System.out.println("Resultset was: "+EcogridResultsetTransformer.toXMLString(_resultsetType)); - } else - { - System.out.println("Resultset was NULL!"); - } - - if (_resultsetType != null) - { - _records = _resultsetType.getRecord(); - if (_records != null) - { - resultsetWasOK = true; - DSTableDef table = new DSTableDef("DarwinCore"); - ResultsetType_record_returnField[] fields = _records[0].getReturnField(); - for (int i=0;i "+dcSchema.lookupTypeFromName(fields[i].getName())); - table.addField(fields[i].getName(), DarwinCoreSchema.lookupTypeFromName(fields[i].getName())); - } - - // create a schemaDef that was used by the results - DSSchemaDef schema = new DSSchemaDef(); - schema.addTable(table); - _resultsetSchemaDef = schema; - - try - { - //DSTableIFace table = (DSTableIFace)_darwinCoreSchema.getTables().elementAt(0); - String sqlDef = _sqlAttr.getExpression(); - if (sqlDef.length() == 0 || _queryDef == null || _columns == null || _colTypes == null) - { - - _columns = table.getFields(); - String initFromData = getInitFromData(); - if (initFromData != null && initFromData.equals("yes")) - { - reconfigurePortsAsField(table.getFields(), _colTypes); - } - } else - { - reconfigurePorts(_queryDef.getSelects(), _colTypes); - } - generateTable(); - - } catch (Exception e) - { - e.printStackTrace(); - } - } - } - } - else - { - setIconStatus(TITLE_ERROR, MAGENTA); - System.err.println("File " + cachedFileName + " is empty."); - } - } - else + _outputType = _ASROW; + } + else if (strOutputType.equals("As Field")) { - setIconStatus(TITLE_ERROR, MAGENTA); - System.err.println("Cached File Name (LocalName) is null or empty!"); + _outputType = _ASFIELD; } - } catch (Exception e) + } + catch (IllegalActionException e) { + setIconStatus(TITLE_ERROR, MAGENTA); + return; + } + ResultsetType _resultsetType = _cachedDataItem.getResultset(); + if (_resultsetType == null) + { + setIconStatus(TITLE_ERROR, MAGENTA); + System.out.println("Resultset was NULL!"); + return; + } + + ResultsetType_record[] _records = null; + _records = _resultsetType.getRecord(); + if (_records != null) + { + DSTableDef table = new DSTableDef("DarwinCore"); + ResultsetType_record_returnField[] fields = _records[0].getReturnField(); + for (int i=0;i "+dcSchema.lookupTypeFromName(fields[i].getName())); + table.addField(fields[i].getName(), DarwinCoreSchema.lookupTypeFromName(fields[i].getName())); } - - if (resultsetWasOK) + + // create a schemaDef that was used by the results + DSSchemaDef schema = new DSSchemaDef(); + schema.addTable(table); + _resultsetSchemaDef = schema; + + try { - DataCacheManager.getInstance().saveCache(); - } else + //DSTableIFace table = (DSTableIFace)_darwinCoreSchema.getTables().elementAt(0); + String sqlDef = _sqlAttr.getExpression(); + if (sqlDef.length() == 0 || _queryDef == null || _columns == null || _colTypes == null) + { + + _columns = table.getFields(); + if (isInitFromData()) + { + reconfigurePortsAsField(table.getFields(), _colTypes); + } + } else + { + reconfigurePorts(_queryDef.getSelects(), _colTypes); + } + + } catch (IllegalActionException e) { - setIconStatus(TITLE_ERROR, MAGENTA); + e.printStackTrace(); } - } else - { - setIconStatus(TITLE_ERROR, MAGENTA); } - + + generateTable( _records ); + setIconStatus(TITLE_BINARY, YELLOW); + } - + } Index: DarwinCoreMetaDataSpecification.java =================================================================== RCS file: /cvs/kepler/src/org/ecoinformatics/seek/datasource/darwincore/DarwinCoreMetaDataSpecification.java,v retrieving revision 1.15 diff -u -r1.15 DarwinCoreMetaDataSpecification.java --- DarwinCoreMetaDataSpecification.java 8 Sep 2005 21:28:32 -0000 1.15 +++ DarwinCoreMetaDataSpecification.java 13 Sep 2005 15:45:21 -0000 @@ -80,7 +80,10 @@ private static final String QUERYID = "darwincore-quick-search-query"; private String _searchString = ""; - private boolean _returnResultsBySpecies = true; + + // compile time configuration for partitioning the resultset into different datasources. + // if set to true, partition by species. If set to false, partition by digir provider url. + private static final boolean _returnResultsBySpecies = true; private int _numResults = 0; /** @@ -232,41 +235,43 @@ Vector aResultList) throws NameDuplicationException, IllegalActionException { + _numResults = 0; if (aResults == null) { return false; } - int numAdded = 0; ResultsetType_resultsetMetadata metaData = aResults.getResultsetMetadata(); - if (metaData != null) + if (metaData == null) { - ResultsetType_resultsetMetadata_system[] system = metaData.getSystem(); - if (system != null) + return false; + } + + ResultsetType_resultsetMetadata_system[] system = metaData.getSystem(); + if (system == null) + { + return false; + } + for (int i=0;i 0; + + return _numResults > 0; } /** @@ -309,6 +314,7 @@ Vector aResultList) throws NameDuplicationException, IllegalActionException { + _numResults = 0; if (aResults == null) { return false; @@ -318,14 +324,12 @@ Hashtable systemProviderHash = new Hashtable(); // collect all the providers into a hash table - int numAdded = 0; ResultsetType_resultsetMetadata metaData = aResults.getResultsetMetadata(); if (metaData != null) { ResultsetType_resultsetMetadata_system[] system = metaData.getSystem(); if (system != null) { - String providerURI = ""; for (int i=0;i 0; + return _numResults > 0; } /**