Metacat

/**

 *  '$RCSfile$'

 *  Copyright: 2007 Regents of the University of California and the

 *             National Center for Ecological Analysis and Synthesis

 *

 *   '$Author$'

 *     '$Date$'

 * '$Revision$'

 *

 * This program is free software; you can redistribute it and/or modify

 * it under the terms of the GNU General Public License as published by

 * the Free Software Foundation; either version 2 of the License, or

 * (at your option) any later version.

 *

 * This program is distributed in the hope that it will be useful,

 * but WITHOUT ANY WARRANTY; without even the implied warranty of

 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

 * GNU General Public License for more details.

 *

 * You should have received a copy of the GNU General Public License

 * along with this program; if not, write to the Free Software

 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA

 */

package edu.ucsb.nceas.metacat;

import java.io.File;

import java.sql.ResultSet;

import java.sql.SQLException;

import java.util.TimerTask;

import org.apache.log4j.Logger;

import edu.ucsb.nceas.metacat.database.DBConnectionPool;

 * A Sitemap represents a document that lists all of the content of the Metacat

 * server for use by harvesting spiders that wish to index the contents of the

 * Metacat site. It is used to generate an XML representation of all of the URLs

 * of the site in order to facilitate indexing of the metacat site by search

 * engines.

 *

 * @author Matt Jones

 */

public class Sitemap extends TimerTask {

	private static Logger logMetacat = Logger.getLogger(Sitemap.class);

    /**

     * Construct a new instance of the Sitemap.

     *

     * @param directory

     *            the location to store sitemap files

     * @param urlRoot

     *            the base URL for constructing sitemap URLs

     * @param skin

     *            the format skin to be used in URLs

     */

    public Sitemap(File directory, String urlRoot, String skin) {

        super();

        this.directory = directory;

        this.urlRoot = urlRoot;

        this.skin = skin;

    }

    /**

     * Execute the timed task when called, in this case by generating the

     * sitemap files needed for this Metacat instance.

     */

    public void run() {

        generateSitemaps();

    }

    /**

     * Generate all of the sitemap files needed to list the URLs from this

     * instance of Metacat, using the open sitemap format described here:

     *

     * URLs are written to a single file, unless the maximum number of URLs

     * allowed inthe sitemap file is exceeded, in which subsequent numbered

     * files are created.

     *

     * @param directory

     *            an existing File directory in which to write the sitemaps

     * @param urlRoot

     *            the base URL to use in constructing document URLs

     * @param skin

     *            the name of the skin to be used in formatting metacat

     *            documents

     */

    public void generateSitemaps() {

        logMetacat.info("Running the Sitemap task.");

        // Test if the passed in File is a directory

        if (directory.isDirectory()) {

            // Query xml_documents to get list of documents

            StringBuffer query = new StringBuffer();

            // TODO: make the doctype configurable in the query

            	"SELECT xml_documents.docid, xml_documents.rev " +

                "AND xml_documents.rev = identifier.rev " +

                "AND identifier.guid = xml_access.guid " +

                "AND xml_access.perm_type = 'allow' " +

                "order by docid, rev";

            query.append(sql);

            DBConnection dbConn = null;

            int serialNumber = -1;

            try {

                // Get a database connection from the pool

                dbConn = DBConnectionPool

                        .getDBConnection("Sitemap.generateSitemap()");

                serialNumber = dbConn.getCheckOutSerialNumber();

                // Execute the query statement

                ResultSet rs = stmt.getResultSet();

                // Loop through all of the documents, and write them to a

                // sitemap

                File sitemapFile = null;

                int fileNumber = 0;

                while (rs.next()) {

                    // Check if a new sitemap file needs to be created

                    if (counter % MAX_URLS_IN_FILE == 0) {

                        // if a sitemap file is already open

                        if (sitemapFile != null && sitemapFile.canWrite()) {

                            // write the footer and close the file

                            writeSitemapFooter(sitemap);

                        }

                        // Open a new sitemap file for writing

                        fileNumber++;

                        sitemapFile = new File(directory, fileRoot + fileNumber

                                + ".xml");

                        // Write the sitemap document header for the new file

                        writeSitemapHeader(sitemap);

                    }

                            + rs.getString(2);

                    writeSitemapEntry(sitemap, docid);

                    counter++;

                }

                stmt.close();

                writeSitemapFooter(sitemap);

                logMetacat.warn("Error while writing to the sitemap file: "

                        + e.getMessage());

            } catch (IOException ioe) {

                logMetacat.warn("Could not open or write to the sitemap file."

                        + ioe.getMessage());

                logMetacat.warn("Could not retrieve the account number separator."

                        + pnfe.getMessage());

                // Return database connection to the pool

                DBConnectionPool.returnDBConnection(dbConn, serialNumber);

            }

        } else {

            logMetacat.warn("Sitemap not created because directory not valid.");

        }

    }

    /**

     * Write the header information in a single sitemap file. This includes the

     * XML prolog, the root element and namespace declaration, and the elements

     * leading up to the first URL entry.

     *

     * @param sitemap

     *            the Writer to use for writing the header

     * @throws IOException

     *             if there is a problem writing to the Writer

     */

    private void writeSitemapHeader(Writer sitemap) throws IOException {

        sitemap.write(PROLOG);

                "xmlns:sm=\"http://www.sitemaps.org/schemas/sitemap/0.9\"\n" +

                "xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\"\n" +

        sitemap.flush();

    }

    /**

     * Write a URL entry to a single sitemap file. This includes the XML markup

     * surrounding a particular site URL.

     *

     * @param sitemap

     *            the Writer to use for writing the URL

     * @param docid

     *            the identifier to be written in the URL

     * @param urlRoot

     *            the base URL to be used in constructing a URL

     * @param skin

     *            the name of the skin to be used in constructing a URL

     * @throws IOException

     *             if there is a problem writing to the Writer

     */

    private void writeSitemapEntry(Writer sitemap, String docid)

            throws IOException {

        if (sitemap != null && docid != null && urlRoot != null) {

            StringBuffer url = new StringBuffer();

            url.append(urlRoot);

            if (!urlRoot.endsWith("/")) {

                url.append("/");

            }

            url.append(docid);

            if (skin != null) {

                url.append("/");

                url.append(skin);

            }

            sitemap.write("<url><loc>");

            sitemap.write(url.toString());

            sitemap.write("</loc>");

            // <lastmod>2005-01-01</lastmod>

            // <changefreq>monthly</changefreq>

            // <priority>0.8</priority>

            sitemap.write("</url>");

            sitemap.write("\n");

            sitemap.flush();

        }

    }

    /**

     * Write the footer information in a single sitemap file and close the file.

     * This includes the closing tag for the root element.

     *

     * @param sitemap

     *            the Writer to use for writing the footer

     * @throws IOException

     *             if there is a problem writing to the Writer

     */

    private void writeSitemapFooter(Writer sitemap) throws IOException {

        {

	    	String footer = "</urlset>\n";

	        sitemap.write(footer);

	        sitemap.close();

        }

     * Create an index file listing all of the sitemap files that were created.

     * @param fileNumber the number of sitemap files that were created.

     */

    private void writeSitemapIndex(int fileNumber) {

        // Open a new sitemapIndex file for writing

        File sitemapIndexFile = null;

        OutputStreamWriter sitemapIndex = null;

        sitemapIndexFile = new File(directory, indexFilename);

        try {

            sitemapIndex = new OutputStreamWriter(new FileOutputStream(sitemapIndexFile), Charset.forName("UTF-8"));

            // Write the sitemap index header for the new file

            sitemapIndex.write(PROLOG);

            String header = "<sitemapindex xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\"\n"

                    + "xmlns:sm=\"http://www.sitemaps.org/schemas/sitemap/0.9\"\n" + "xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\"\n"

                    + "xsi:schemaLocation=\"http://www.sitemaps.org/schemas/sitemap/0.9 http://www.sitemaps.org/schemas/sitemap/0.9/siteindex.xsd\">\n";

            sitemapIndex.write(header);

            sitemapIndex.flush();

            // Write out one index entry for each sitemap file

            for (int fn = 1; fn <= fileNumber; fn++) {

                String filename = fileRoot + fileNumber + ".xml";

                writeSitemapIndexEntry(sitemapIndex, filename);

            }

            // Write the sitemap index footer content

            if (sitemapIndex != null) {

                String footer = "</sitemapindex>\n";

                sitemapIndex.write(footer);

                sitemapIndex.close();

            }

            // Close the index file

            if (sitemapIndex != null) {

                sitemapIndex.close();

            }

        } catch (IOException e) {

            logMetacat.warn("Could not open or write to the sitemap index file." + e.getMessage());

        }

    }

    /**

     * Write a single line of the sitemap index file containing the URL to a specific sitemap file.

     * @param sitemapIndex the writer to which the index information is written

     * @param filename the name of the index file to be used

     * @throws IOException on error writing to the index file

     */

    private void writeSitemapIndexEntry(Writer sitemapIndex, String filename)

            throws IOException {

        if (sitemapIndex != null && filename != null && urlRoot != null) {

            StringBuffer url = new StringBuffer();

            url.append(urlRoot);

            if (!urlRoot.endsWith("/")) {

                url.append("/");

            }

            url.append(filename);

            sitemapIndex.write("<sitemap><loc>");

            sitemapIndex.write(url.toString());

            sitemapIndex.write("</loc>");

            sitemapIndex.write("</sitemap>");

            sitemapIndex.write("\n");

            sitemapIndex.flush();

        }

    }

    /** The directory in which sitemaps are written. */

    private File directory;

    /** The root url for constructing sitemap URLs. */

    private String urlRoot;

    /** The name of the format skin to be used in sitemap URLs. */

    private String skin;

    /** Maximum number of URLs to write to a single sitemap file */

    static final int MAX_URLS_IN_FILE = 25000; // 50,000 according to Google

    /** The root name to be used in naming sitemap files. */

    static final String fileRoot = "metacat";

    /** The name to give to the sitemap index file */

    static final String indexFilename = "metacatSitemapIndex.xml";

    /** A String constant containing the XML prolog to be written in files. */

    static final String PROLOG = "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n";

}