[ { "id": "https://authors.library.caltech.edu/records/kbzb2-wpc32", "eprint_id": 28225, "eprint_status": "archive", "datestamp": "2023-08-22 03:49:50", "lastmod": "2023-10-24 17:45:04", "type": "book_section", "metadata_visibility": "show", "creators": { "items": [ { "id": "Djorgovski-S-G", "name": { "family": "Djorgovski", "given": "S. G." }, "orcid": "0000-0002-0603-3087" } ] }, "title": "Virtual Astronomy, Information Technology, and the New Scientific Methodology", "ispublished": "unpub", "full_text_status": "public", "keywords": "Astronomy; Data management; Information technology; Knowledge acquisition; Knowledge representation; Scientific visualization", "note": "\u00a9 2005 IEEE.\n\nManuscript submitted on March 7, 2005. \n\nThis work was supported in part by the U.S. National Science Foundation grants AST-0122449, AST-0326524, AST-0407448, DMS-0101360, NASA contract NAG5-9482, and the Ajax Foundation.\n\n
Published - 01508175.pdf
Submitted - 115.pdf
", "abstract": "All sciences, including astronomy, are now entering the era of information abundance. The exponentially increasing volume and complexity of modern data sets promises to transform the scientific practice, but also poses a number of common technological challenges. The Virtual Observatory concept is the astronomical community's response to these challenges: it aims to harness the progress in information technology in the service of astronomy, and at the same time provide a valuable testbed for information technology and applied computer science. Challenges broadly fall into two categories: data handling (or \"data farming\"), including issues such as archives, intelligent storage, databases, interoperability, fast networks, etc., and data mining, data understanding, and knowledge discovery, which include issues such as automated clustering and classification, multivariate correlation searches, pattern recognition, visualization in highly hyperdimensional parameter spaces, etc., as well as various applications of machine learning in these contexts. Such techniques are forming a methodological foundation for science with massive and complex data sets in general, and are likely to have a much broather impact on the modern society, commerce, information economy, security, etc. There is a powerful emerging synergy between the\ncomputationally enabled science and the science-driven computing, which will drive the progress in science, scholarship, and many other venues in the 21st century.", "date": "2005-07", "date_type": "published", "publisher": "IEEE", "place_of_pub": "Piscataway, NJ", "pagerange": "125-132", "id_number": "CaltechCACR:2005.115", "isbn": "0-7695-2255-6", "book_title": "Seventh International Workshop on Computer Architecture for Machine Perception (CAMP'05)", "official_url": "https://resolver.caltech.edu/CaltechCACR:2005.115", "rights": "You are granted permission for individual, educational, research and non-commercial reproduction, distribution, display and performance of this work in any format.", "funders": { "items": [ { "agency": "NSF", "grant_number": "AST-0122449" }, { "agency": "NSF", "grant_number": "AST-0326524" }, { "agency": "NSF", "grant_number": "AST-0407448" }, { "agency": "NSF", "grant_number": "DMS-0101360" }, { "agency": "NASA", "grant_number": "NAG5-9482" }, { "agency": "Ajax Foundation" } ] }, "local_group": { "items": [ { "id": "Center-for-Advanced-Computing-Research" } ] }, "contributors": { "items": [ { "id": "Di-Gesu-V", "name": { "family": "Di Gesu", "given": "V." } }, { "id": "Tegolo-D", "name": { "family": "Tegolo", "given": "D." } } ] }, "doi": "10.1109/CAMP.2005.53", "primary_object": { "basename": "01508175.pdf", "url": "https://authors.library.caltech.edu/records/kbzb2-wpc32/files/01508175.pdf" }, "related_objects": [ { "basename": "115.pdf", "url": "https://authors.library.caltech.edu/records/kbzb2-wpc32/files/115.pdf" } ], "pub_year": "2005", "author_list": "Djorgovski, S. G." }, { "id": "https://authors.library.caltech.edu/records/19j6v-4my77", "eprint_id": 28227, "eprint_status": "archive", "datestamp": "2023-08-19 15:04:02", "lastmod": "2024-01-13 05:49:09", "type": "book_section", "metadata_visibility": "show", "creators": { "items": [ { "id": "Jacob-J-C", "name": { "family": "Jacob", "given": "Joseph C." } }, { "id": "Katz-D-S", "name": { "family": "Katz", "given": "Daniel S." } }, { "id": "Miller-C-D", "name": { "family": "Miller", "given": "Craig D." } }, { "id": "Walia-H", "name": { "family": "Walia", "given": "Harshpreet" } }, { "id": "Williams-R", "name": { "family": "Williams", "given": "Roy" } }, { "id": "Djorgovski-S-G", "name": { "family": "Djorgovski", "given": "S. George" }, "orcid": "0000-0002-0603-3087" }, { "id": "Graham-M-J", "name": { "family": "Graham", "given": "Matthew" }, "orcid": "0000-0002-3168-0139" }, { "id": "Mahabal-A-A", "name": { "family": "Mahabal", "given": "Ashish" }, "orcid": "0000-0003-2242-0244" }, { "id": "Babu-J", "name": { "family": "Babu", "given": "Jogesh" } }, { "id": "Vanden-Berk-D-E", "name": { "family": "Vanden Berk", "given": "Daniel E." } }, { "id": "Nichol-R", "name": { "family": "Nichol", "given": "Robert" } } ] }, "title": "Grist: Grid-based Data Mining for Astronomy", "ispublished": "unpub", "full_text_status": "public", "note": "Part of this research was carried out at the Jet Propulsion Laboratory, California Institute of Technology, and was sponsored by the National Science Foundation through an agreement with the National Aeronautics and Space Administration. \n\nAlso available: arXiv:astro-ph/0411589 v1 19 Nov 2004\n\nSubmitted - 118.pdf
", "abstract": "The Grist project is developing a grid-technology based system as a research environment for astronomy with massive and complex datasets. This knowledge extraction system will consist of a library of distributed grid services controlled by a work ow system, compliant with standards emerging from the grid computing, web services, and virtual observatory communities. This new technology is being used to find high redshift quasars, study peculiar variable objects, search for transients in real time, and fit SDSS QSO spectra to measure black hole masses. Grist services are also a component of the \"hyperatlas\" project to serve high-resolution multi-wavelength imagery over the Internet. In support of these science and outreach objectives, the Grist framework will provide the enabling fabric to tie together distributed grid services in the areas of data access, federation, mining, subsetting, source extraction, image mosaicking, statistics, and visualization.", "date": "2005", "date_type": "published", "publisher": "Astronomical Society of the Pacific", "id_number": "CaltechCACR:2005.118", "official_url": "https://resolver.caltech.edu/CaltechCACR:2005.118", "rights": "You are granted permission for individual, educational, research and non-commercial reproduction, distribution, display and performance of this work in any format.", "funders": { "items": [ { "agency": "NASA/JPL/Caltech" }, { "agency": "NSF" } ] }, "local_group": { "items": [ { "id": "Center-for-Advanced-Computing-Research" } ] }, "contributors": { "items": [ { "id": "Shopbell-P-L", "name": { "family": "Shopbell", "given": "P. L." } }, { "id": "Britton-M-C", "name": { "family": "Britton", "given": "M. C." } }, { "id": "Ebert-R-J", "name": { "family": "Ebert", "given": "R." } } ] }, "primary_object": { "basename": "118.pdf", "url": "https://authors.library.caltech.edu/records/19j6v-4my77/files/118.pdf" }, "pub_year": "2005", "author_list": "Jacob, Joseph C.; Katz, Daniel S.; et el." }, { "id": "https://authors.library.caltech.edu/records/tpzkk-gyg45", "eprint_id": 28226, "eprint_status": "archive", "datestamp": "2023-08-19 15:03:57", "lastmod": "2024-01-13 05:49:07", "type": "book_section", "metadata_visibility": "show", "creators": { "items": [ { "id": "Mahabal-A-A", "name": { "family": "Mahabal", "given": "Ashish A." }, "orcid": "0000-0003-2242-0244" }, { "id": "Djorgovski-S-G", "name": { "family": "Djorgovski", "given": "S. G." }, "orcid": "0000-0002-0603-3087" }, { "id": "Graham-M-J", "name": { "family": "Graham", "given": "M. J." }, "orcid": "0000-0002-3168-0139" }, { "id": "Kollipara-P", "name": { "family": "Kollipara", "given": "Priya" } }, { "id": "Granett-B", "name": { "family": "Granett", "given": "Benjamin" } }, { "id": "Krause-E", "name": { "family": "Krause", "given": "Elisabeth" } }, { "id": "Williams-R", "name": { "family": "Williams", "given": "Roy" } }, { "id": "Bogosavljevic-M", "name": { "family": "Bogosavljevic", "given": "M." } }, { "id": "Baltay-C", "name": { "family": "Baltay", "given": "C." } }, { "id": "Rabinowitz-D", "name": { "family": "Rabinowitz", "given": "D." } }, { "id": "Bauer-A", "name": { "family": "Bauer", "given": "A." } }, { "id": "Andrews-P", "name": { "family": "Andrews", "given": "P." } }, { "id": "Ellman-N", "name": { "family": "Ellman", "given": "N." } }, { "id": "Duffau-S", "name": { "family": "Duffau", "given": "S." } }, { "id": "Jerke-J", "name": { "family": "Jerke", "given": "J." } }, { "id": "Rengstorf-A", "name": { "family": "Rengstorf", "given": "A." } }, { "id": "Brunner-R", "name": { "family": "Brunner", "given": "R." } }, { "id": "Musser-J", "name": { "family": "Musser", "given": "J." } }, { "id": "Mufson-S", "name": { "family": "Mufson", "given": "S." } }, { "id": "Gebhard-M", "name": { "family": "Gebhard", "given": "M." } } ] }, "title": "Time Domain Explorations With Digital Sky Surveys", "ispublished": "unpub", "full_text_status": "public", "note": "This work was supported in part by the NSF grants AST-0326524, AST-0407448 and the NASA contract NAG5-9482. PK and EK were supported in part by SURF Fellowships at Caltech. SGD acknowledges a partial support from the Ajax Foundation. \n\nAlso available: arXiv:astro-ph/0412164 v1 7 Dec 2004", "abstract": "One of the new frontiers of astronomical research is the exploration of time variability on the sky at different wavelengths and flux levels. We have carried out a pilot project using DPOSS data to study strong variables and transients, and are now extending it to the new Palomar-QUEST synoptic sky survey. We report on our early findings and outline the methodology to be implemented in preparation for a real-time transient detection pipeline. In addition to large numbers of known types of highly variable sources (e.g., SNe, CVs, OVV QSOs, etc.), we expect to find numerous transients whose nature may be established by a rapid follow-up. Whereas we will make all detected variables publicly available through the web, we anticipate that email alerts would be issued in the real time for a subset of events deemed to be the most interesting. This real-time process entails many challenges, in an effort to maintain a high completeness while keeping the contamination low. We will utilize distributed Grid services developed by the GRIST project, and implement a variety of advanced statistical and machine learning techniques.", "date": "2005", "date_type": "published", "publisher": "Astronomical Society of the Pacific", "id_number": "CaltechCACR:2005.117", "official_url": "https://resolver.caltech.edu/CaltechCACR:2005.117", "rights": "You are granted permission for individual, educational, research and non-commercial reproduction, distribution, display and performance of this work in any format.", "local_group": { "items": [ { "id": "Center-for-Advanced-Computing-Research" } ] }, "contributors": { "items": [ { "name": { "family": "Shopbell", "given": "P. L." } }, { "name": { "family": "Britton", "given": "M. C." } }, { "name": { "family": "Ebert", "given": "R." } } ] }, "primary_object": { "basename": "117.pdf", "url": "https://authors.library.caltech.edu/records/tpzkk-gyg45/files/117.pdf" }, "pub_year": "2005", "author_list": "Mahabal, Ashish A.; Djorgovski, S. G.; et el." }, { "id": "https://authors.library.caltech.edu/records/hgvjr-gm741", "eprint_id": 28193, "eprint_status": "archive", "datestamp": "2023-08-19 10:31:36", "lastmod": "2023-10-24 17:44:06", "type": "book_section", "metadata_visibility": "show", "creators": { "items": [ { "id": "Williams-R", "name": { "family": "Williams", "given": "Roy" } } ] }, "title": "Grids and the Virtual Observatory", "ispublished": "unpub", "full_text_status": "public", "abstract": "We consider several projects from astronomy that benefit from the Grid paradigm and\nassociated technology, many of which involve either massive datasets or the federation\nof multiple datasets. We cover image computation (mosaicking, multi-wavelength\nimages, and synoptic surveys); database computation (representation through XML,\ndata mining, and visualization); and semantic interoperability (publishing, ontologies,\ndirectories, and service descriptions).", "date": "2003", "date_type": "published", "publisher": "Wiley", "id_number": "CaltechCACR:2003.200", "official_url": "https://resolver.caltech.edu/CaltechCACR:2003.200", "rights": "You are granted permission for individual, educational, research and non-commercial reproduction, distribution, display and performance of this work in any format.", "local_group": { "items": [ { "id": "Center-for-Advanced-Computing-Research" } ] }, "contributors": { "items": [ { "name": { "family": "Berman", "given": "Fran" } }, { "name": { "family": "Hey", "given": "Anthony J.G." } }, { "name": { "family": "Fox", "given": "Geoffrey" } } ] }, "primary_object": { "basename": "vogrid.pdf", "url": "https://authors.library.caltech.edu/records/hgvjr-gm741/files/vogrid.pdf" }, "pub_year": "2003", "author_list": "Williams, Roy" }, { "id": "https://authors.library.caltech.edu/records/576sr-fwb54", "eprint_id": 103834, "eprint_status": "archive", "datestamp": "2023-08-21 22:25:33", "lastmod": "2024-01-15 04:31:52", "type": "book_section", "metadata_visibility": "show", "creators": { "items": [ { "id": "Deelman-E", "name": { "family": "Deelman", "given": "Ewa" } }, { "id": "Kesselman-C", "name": { "family": "Kesselman", "given": "Carl" } }, { "id": "Williams-R-D", "name": { "family": "Williams", "given": "Roy" }, "orcid": "0000-0002-9145-8580" }, { "id": "Lazzarini-A", "name": { "family": "Lazzarini", "given": "Albert" } }, { "id": "Prince-T-A", "name": { "family": "Prince", "given": "Thomas A." }, "orcid": "0000-0002-8850-3627" }, { "id": "Romano-J", "name": { "family": "Romano", "given": "Joe" } }, { "id": "Allen-B", "name": { "family": "Allen", "given": "Bruce" } } ] }, "title": "A Virtual Data Grid for LIGO", "ispublished": "unpub", "full_text_status": "restricted", "keywords": "Large Hadron Collider; Gravitational Wave; Virtual Channel; Virtual Data; Request Manager", "note": "\u00a9 2001 Springer-Verlag Berlin Heidelberg. \n\nFirst Online: 12 July 2001.", "abstract": "GriPhyN (Grid Physics Network) is a large US collaboration to build grid services for large physics experiments, one of which is LIGO, a gravitational-wave observatory. This paper explains the physics and computing challenges of LIGO, and the tools that GriPhyN will build to address them. A key component needed to implement the data pipeline is a virtual data service; a system to dynamically create data products requested during the various stages. The data could possibly be already processed in a certain way, it may be in a file on a storage system, it may be cached, or it may need to be created through computation. The full elaboration of this system will allow complex data pipelines to be set up as virtual data objects, with existing data being transformed in diverse ways.", "date": "2001-07-12", "date_type": "published", "publisher": "Springer", "place_of_pub": "Berlin", "pagerange": "3-12", "id_number": "CaltechAUTHORS:20200611-070141219", "isbn": "978-3-540-42293-8", "book_title": "High-Performance Computing and Networking", "official_url": "https://resolver.caltech.edu/CaltechAUTHORS:20200611-070141219", "rights": "No commercial reproduction, distribution, display or performance rights in this work are provided.", "local_group": { "items": [ { "id": "Center-for-Advanced-Computing-Research" }, { "id": "Space-Radiation-Laboratory" } ] }, "contributors": { "items": [ { "id": "Hertzberger-B", "name": { "family": "Hertzberger", "given": "Bob" } }, { "id": "Hoekstra-A", "name": { "family": "Hoekstra", "given": "Alfons" } }, { "id": "Williams-R-D", "name": { "family": "Williams", "given": "Roy D." } } ] }, "doi": "10.1007/3-540-48228-8_1", "pub_year": "2001", "author_list": "Deelman, Ewa; Kesselman, Carl; et el." }, { "id": "https://authors.library.caltech.edu/records/2y3b1-2s843", "eprint_id": 104306, "eprint_status": "archive", "datestamp": "2023-08-22 13:26:24", "lastmod": "2024-01-15 04:32:13", "type": "book_section", "metadata_visibility": "show", "creators": { "items": [ { "id": "Blackburn-K", "name": { "family": "Blackburn", "given": "Kent" }, "orcid": "0000-0002-3838-2986" }, { "id": "Lazzarini-A", "name": { "family": "Lazzarini", "given": "Albert" } }, { "id": "Prince-T-A", "name": { "family": "Prince", "given": "Tom" }, "orcid": "0000-0002-8850-3627" }, { "id": "Williams-R-D", "name": { "family": "Williams", "given": "Roy" }, "orcid": "0000-0002-9145-8580" } ] }, "title": "XSIL: Extensible scientific interchange language", "ispublished": "unpub", "full_text_status": "restricted", "keywords": "Synthetic Aperture Radar; Binary File; Document Type Definition; Stream Element; Frame Object", "note": "\u00a9 1999 Springer-Verlag.", "abstract": "We motivate and define the XSIL language as a flexible, hierarchical, extensible transport language for scientific data objects. The entire object may be represented in the file, or there may be metadata in the XSIL file, with a powerful, fault-tolerant linking mechanism to external data. The language is based on XML, and is designed not only for parsing and processing by machines, but also for presentation to humans through web browsers and web-database technology. There is a natural mapping between the elements of the XSIL language and the object model into which they are translated by the parser. As well as common objects (Parameter, Array, Time, Table), we have extended XSIL to include the IGWDFrame, used by gravitational-wave observatories.", "date": "1999", "date_type": "published", "publisher": "Springer", "place_of_pub": "Berlin", "pagerange": "513-524", "id_number": "CaltechAUTHORS:20200708-152256926", "isbn": "9783540658214", "book_title": "High-Performance Computing and Networking", "official_url": "https://resolver.caltech.edu/CaltechAUTHORS:20200708-152256926", "rights": "No commercial reproduction, distribution, display or performance rights in this work are provided.", "local_group": { "items": [ { "id": "Center-for-Advanced-Computing-Research" }, { "id": "LIGO" } ] }, "contributors": { "items": [ { "id": "Sloot-P", "name": { "family": "Sloot", "given": "Peter" } }, { "id": "Bubak-M", "name": { "family": "Bubak", "given": "Marian" } }, { "id": "Hoekstra-A", "name": { "family": "Hoekstra", "given": "Alfons" } }, { "id": "Hertzberger-B", "name": { "family": "Hertzberger", "given": "Bob" } } ] }, "doi": "10.1007/bfb0100612", "pub_year": "1999", "author_list": "Blackburn, Kent; Lazzarini, Albert; et el." } ]