[{"related_material":{"record":[{"id":"12856","status":"public","relation":"later_version"}]},"file":[{"file_name":"main.pdf","file_size":662409,"date_created":"2023-01-27T03:18:34Z","checksum":"55426e463fdeafe9777fc3ff635154c7","file_id":"12408","date_updated":"2023-01-27T03:18:34Z","success":1,"creator":"fmuehlbo","access_level":"open_access","relation":"main_file","content_type":"application/pdf"}],"title":"VAMOS: Middleware for Best-Effort Third-Party Monitoring","alternative_title":["IST Austria Technical Report"],"has_accepted_license":"1","project":[{"call_identifier":"H2020","grant_number":"101020093","_id":"62781420-2b32-11ec-9570-8d9b63373d4d","name":"Vigilant Algorithmic Monitoring of Software"}],"citation":{"chicago":"Chalupa, Marek, Fabian Mühlböck, Stefanie Muroya Lei, and Thomas A Henzinger. <i>VAMOS: Middleware for Best-Effort Third-Party Monitoring</i>. Institute of Science and Technology Austria, 2023. <a href=\"https://doi.org/10.15479/AT:ISTA:12407\">https://doi.org/10.15479/AT:ISTA:12407</a>.","ieee":"M. Chalupa, F. Mühlböck, S. Muroya Lei, and T. A. Henzinger, <i>VAMOS: Middleware for Best-Effort Third-Party Monitoring</i>. Institute of Science and Technology Austria, 2023.","mla":"Chalupa, Marek, et al. <i>VAMOS: Middleware for Best-Effort Third-Party Monitoring</i>. Institute of Science and Technology Austria, 2023, doi:<a href=\"https://doi.org/10.15479/AT:ISTA:12407\">10.15479/AT:ISTA:12407</a>.","ista":"Chalupa M, Mühlböck F, Muroya Lei S, Henzinger TA. 2023. VAMOS: Middleware for Best-Effort Third-Party Monitoring, Institute of Science and Technology Austria, 38p.","short":"M. Chalupa, F. Mühlböck, S. Muroya Lei, T.A. Henzinger, VAMOS: Middleware for Best-Effort Third-Party Monitoring, Institute of Science and Technology Austria, 2023.","apa":"Chalupa, M., Mühlböck, F., Muroya Lei, S., &#38; Henzinger, T. A. (2023). <i>VAMOS: Middleware for Best-Effort Third-Party Monitoring</i>. Institute of Science and Technology Austria. <a href=\"https://doi.org/10.15479/AT:ISTA:12407\">https://doi.org/10.15479/AT:ISTA:12407</a>","ama":"Chalupa M, Mühlböck F, Muroya Lei S, Henzinger TA. <i>VAMOS: Middleware for Best-Effort Third-Party Monitoring</i>. Institute of Science and Technology Austria; 2023. doi:<a href=\"https://doi.org/10.15479/AT:ISTA:12407\">10.15479/AT:ISTA:12407</a>"},"ec_funded":1,"abstract":[{"lang":"eng","text":"As the complexity and criticality of software increase every year, so does the importance of run-time monitoring. Third-party monitoring, with limited knowledge of the monitored software, and best-effort monitoring, which keeps pace with the monitored software, are especially valuable, yet underexplored areas of run-time monitoring. Most existing monitoring frameworks do not support their combination because they either require access to the monitored code for instrumentation purposes or the processing of all observed events, or both.\r\n\r\nWe present a middleware framework, VAMOS, for the run-time monitoring of software which is explicitly designed to support third-party and best-effort scenarios. The design goals of VAMOS are (i) efficiency (keeping pace at low overhead), (ii) flexibility (the ability to monitor black-box code through a variety of different event channels, and the connectability to monitors written in different specification languages), and (iii) ease-of-use. To achieve its goals, VAMOS combines aspects of event broker and event recognition systems with aspects of stream processing systems.\r\n\r\nWe implemented a prototype toolchain for VAMOS and conducted experiments including a case study of monitoring for data races. The results indicate that VAMOS enables writing useful yet efficient monitors, is compatible with a variety of event sources and monitor specifications, and simplifies key aspects of setting up a monitoring system from scratch."}],"oa_version":"Published Version","page":"38","publication_status":"published","day":"27","tmp":{"legal_code_url":"https://creativecommons.org/licenses/by/4.0/legalcode","image":"/images/cc_by.png","name":"Creative Commons Attribution 4.0 International Public License (CC-BY 4.0)","short":"CC BY (4.0)"},"license":"https://creativecommons.org/licenses/by/4.0/","status":"public","date_updated":"2023-04-25T07:19:06Z","publication_identifier":{"eissn":["2664-1690"]},"month":"01","keyword":["runtime monitoring","best effort","third party"],"article_processing_charge":"No","ddc":["005"],"user_id":"2DF688A6-F248-11E8-B48F-1D18A9856A87","file_date_updated":"2023-01-27T03:18:34Z","acknowledgement":"This work was supported in part by the ERC-2020-AdG 101020093. \r\nThe authors would like to thank the anonymous FASE reviewers for their valuable feedback and suggestions.","department":[{"_id":"ToHe"}],"date_created":"2023-01-27T03:18:08Z","date_published":"2023-01-27T00:00:00Z","oa":1,"_id":"12407","author":[{"first_name":"Marek","full_name":"Chalupa, Marek","id":"87e34708-d6c6-11ec-9f5b-9391e7be2463","last_name":"Chalupa"},{"full_name":"Mühlböck, Fabian","first_name":"Fabian","id":"6395C5F6-89DF-11E9-9C97-6BDFE5697425","last_name":"Mühlböck","orcid":"0000-0003-1548-0177"},{"id":"a376de31-8972-11ed-ae7b-d0251c13c8ff","last_name":"Muroya Lei","full_name":"Muroya Lei, Stefanie","first_name":"Stefanie"},{"full_name":"Henzinger, Thomas A","first_name":"Thomas A","id":"40876CD8-F248-11E8-B48F-1D18A9856A87","last_name":"Henzinger","orcid":"0000-0002-2985-7724"}],"type":"technical_report","publisher":"Institute of Science and Technology Austria","year":"2023","language":[{"iso":"eng"}],"doi":"10.15479/AT:ISTA:12407"},{"main_file_link":[{"open_access":"1","url":"https://arxiv.org/abs/2009.06429"}],"publication":"21st International Conference on Runtime Verification","external_id":{"isi":["000719383800003"],"arxiv":["2009.06429"]},"related_material":{"record":[{"status":"public","id":"13234","relation":"extended_version"}]},"title":"Into the unknown: active monitoring of neural networks","project":[{"grant_number":"754411","call_identifier":"H2020","name":"ISTplus - Postdoctoral Fellowships","_id":"260C2330-B435-11E9-9278-68D0E5697425"},{"name":"The Wittgenstein Prize","_id":"25F42A32-B435-11E9-9278-68D0E5697425","grant_number":"Z211","call_identifier":"FWF"}],"arxiv":1,"citation":{"ieee":"A. Lukina, C. Schilling, and T. A. Henzinger, “Into the unknown: active monitoring of neural networks,” in <i>21st International Conference on Runtime Verification</i>, Virtual, 2021, vol. 12974, pp. 42–61.","chicago":"Lukina, Anna, Christian Schilling, and Thomas A Henzinger. “Into the Unknown: Active Monitoring of Neural Networks.” In <i>21st International Conference on Runtime Verification</i>, 12974:42–61. Cham: Springer Nature, 2021. <a href=\"https://doi.org/10.1007/978-3-030-88494-9_3\">https://doi.org/10.1007/978-3-030-88494-9_3</a>.","ama":"Lukina A, Schilling C, Henzinger TA. Into the unknown: active monitoring of neural networks. In: <i>21st International Conference on Runtime Verification</i>. Vol 12974. Cham: Springer Nature; 2021:42-61. doi:<a href=\"https://doi.org/10.1007/978-3-030-88494-9_3\">10.1007/978-3-030-88494-9_3</a>","ista":"Lukina A, Schilling C, Henzinger TA. 2021. Into the unknown: active monitoring of neural networks. 21st International Conference on Runtime Verification. RV: Runtime Verification, LNCS, vol. 12974, 42–61.","apa":"Lukina, A., Schilling, C., &#38; Henzinger, T. A. (2021). Into the unknown: active monitoring of neural networks. In <i>21st International Conference on Runtime Verification</i> (Vol. 12974, pp. 42–61). Cham: Springer Nature. <a href=\"https://doi.org/10.1007/978-3-030-88494-9_3\">https://doi.org/10.1007/978-3-030-88494-9_3</a>","mla":"Lukina, Anna, et al. “Into the Unknown: Active Monitoring of Neural Networks.” <i>21st International Conference on Runtime Verification</i>, vol. 12974, Springer Nature, 2021, pp. 42–61, doi:<a href=\"https://doi.org/10.1007/978-3-030-88494-9_3\">10.1007/978-3-030-88494-9_3</a>.","short":"A. Lukina, C. Schilling, T.A. Henzinger, in:, 21st International Conference on Runtime Verification, Springer Nature, Cham, 2021, pp. 42–61."},"scopus_import":"1","alternative_title":["LNCS"],"oa_version":"Preprint","publication_status":"published","page":"42-61","day":"06","ec_funded":1,"abstract":[{"text":"Neural-network classifiers achieve high accuracy when predicting the class of an input that they were trained to identify. Maintaining this accuracy in dynamic environments, where inputs frequently fall outside the fixed set of initially known classes, remains a challenge. The typical approach is to detect inputs from novel classes and retrain the classifier on an augmented dataset. However, not only the classifier but also the detection mechanism needs to adapt in order to distinguish between newly learned and yet unknown input classes. To address this challenge, we introduce an algorithmic framework for active monitoring of a neural network. A monitor wrapped in our framework operates in parallel with the neural network and interacts with a human user via a series of interpretable labeling queries for incremental adaptation. In addition, we propose an adaptive quantitative monitor to improve precision. An experimental evaluation on a diverse set of benchmarks with varying numbers of classes confirms the benefits of our active monitoring framework in dynamic scenarios.","lang":"eng"}],"date_updated":"2024-01-30T12:06:56Z","status":"public","month":"10","quality_controlled":"1","publication_identifier":{"isbn":["9-783-0308-8493-2"],"eisbn":["978-3-030-88494-9"],"issn":["0302-9743"],"eissn":["1611-3349"]},"isi":1,"volume":"12974 ","date_created":"2021-10-31T23:01:31Z","department":[{"_id":"ToHe"}],"acknowledgement":"We thank Christoph Lampert and Alex Greengold for fruitful discussions. This research was supported in part by the Simons Institute for the Theory of Computing, the Austrian Science Fund (FWF) under grant Z211-N23 (Wittgenstein Award), and the European Union’s Horizon 2020 research and innovation programme under the Marie Skłodowska-Curie grant agreement No. 754411.","user_id":"4359f0d1-fa6c-11eb-b949-802e58b17ae8","article_processing_charge":"No","conference":{"name":"RV: Runtime Verification","end_date":"2021-10-14","start_date":"2021-10-11","location":"Virtual"},"place":"Cham","keyword":["monitoring","neural networks","novelty detection"],"author":[{"full_name":"Lukina, Anna","first_name":"Anna","last_name":"Lukina","id":"CBA4D1A8-0FE8-11E9-BDE6-07BFE5697425"},{"first_name":"Christian","full_name":"Schilling, Christian","last_name":"Schilling","id":"3A2F4DCE-F248-11E8-B48F-1D18A9856A87","orcid":"0000-0003-3658-1065"},{"first_name":"Thomas A","full_name":"Henzinger, Thomas A","orcid":"0000-0002-2985-7724","id":"40876CD8-F248-11E8-B48F-1D18A9856A87","last_name":"Henzinger"}],"oa":1,"_id":"10206","date_published":"2021-10-06T00:00:00Z","language":[{"iso":"eng"}],"doi":"10.1007/978-3-030-88494-9_3","publisher":"Springer Nature","year":"2021","type":"conference"}]