@article {Simula.simula.2804,
	title = {Audiovisual robustness: Exploring perceptual tolerance to asynchrony and quality distortion},
	journal = {Multimedia Tools and Applications},
	volume = {74},
	year = {2015},
	pages = {345-365},
	publisher = {Springer},
	abstract = {Rules-of-thumb for noticeable and detrimental asynchrony between audio and video streams have long since been established from the contributions of several studies. Although these studies share similar findings, none have made any discernible assumptions regarding audio and video quality. Considering the use of active adaptation in present and upcoming streaming systems, audio and video will continue to be delivered in separate streams; consequently, the assumption that the rules-of-thumb hold independent of quality needs to be challenged. To put this assumption to the test, we focus on the detection, not the appraisal, of asynchrony at different levels of distortion. Cognitive psychologists use the term temporal integration to describe the failure to detect asynchrony. The term refers to a perceptual process with an inherent buffer for short asynchronies, where corresponding auditory and visual signals are merged into one experience. Accordingly, this paper discusses relevant causes and concerns with regards to asynchrony, it introduces research on audiovisual perception, and it moves on to explore the impact of audio and video quality on the temporal integration of different audiovisual events. Three content types are explored, speech from a news broadcast, music presented by a drummer, and physical action in the form of a chess game. Within these contexts, we found temporal integration to be very robust to quality discrepancies between the two modalities. In fact, asynchrony detection thresholds varied considerably more between the different content than they did between distortion levels. Nevertheless, our findings indicate that the assumption concerning the independence of asynchrony and audiovisual quality may have to be reconsidered.},
	keywords = {Workshop},
	author = {Eg, Ragnhild and Griwodz, Carsten and Halvorsen, P{\r a}l and Dawn Marie {Behne}}
}
@article {23523,
	title = {Audiovisual temporal integration in reverberant environments},
	journal = {Speech Communication},
	volume = {66},
	year = {2015},
	pages = {91-106},
	publisher = {Elsevier},
	abstract = {
p, li { white-space: pre-wrap; }

With teleconferencing becoming more accessible as a communication platform, researchers are working to understand the consequences of the interaction between human perception and this unfamiliar environment. Given the enclosed space of a teleconference room, along with the physical separation between the user, microphone and speakers, the transmitted audio often becomes mixed with the reverberating auditory components from the room. As a result, the audio can be perceived as smeared in time, and this can affect the user experience and perceived quality. Moreover, other challenges remain to be solved. For instance, during encoding, compression and transmission, the audio and video streams are typically treated separately. Consequently, the signals are rarely perfectly aligned and synchronous. In effect, timing affects both reverberation and audiovisual synchrony, and the two challenges may well be inter-dependent. This study explores the temporal integration of audiovisual continuous speech and speech syllables, along with a non-speech event, across a range of asynchrony levels for different reverberation conditions. Non-reverberant stimuli are compared to stimuli with added reverberation recordings. Findings reveal that reverberation does not affect the temporal integration of continuous speech. However, reverberation influences the temporal integration of the isolated speech syllables and the action-oriented event, with perceived subjective synchrony skewed towards audio lead asynchrony and away from the more common audio lag direction. Furthermore, less time is spent on simultaneity judgements for the longer sequences when the temporal offsets get longer and when reverberation is introduced, suggesting that both asynchrony and reverberation add to the demands of the task.},
	keywords = {audiovisual asynchrony, audiovisual speech, reverberation, temporal integration},
	doi = {10.1016/j.specom.2014.10.001},
	url = {http://dx.doi.org/10.1016/j.specom.2014.10.001},
	author = {Eg, Ragnhild and Dawn Marie {Behne} and Griwodz, Carsten}
}
@article {23522,
	title = {Perceived synchrony for realistic and dynamic audiovisual events},
	journal = {Frontiers in Psychology},
	volume = {6},
	year = {2015},
	pages = {1-12},
	publisher = {Frontiers},
	abstract = {In well-controlled laboratory experiments, researchers have found that humans can perceive delays between auditory and visual signals as short as 20 ms. Conversely, other experiments have shown that humans can tolerate audiovisual asynchrony that exceeds 200 ms. This seeming contradiction in human temporal sensitivity can be attributed to a number of factors such as experimental approaches and precedence of the asynchronous signals, along with the nature, duration, location, complexity and repetitiveness of the audiovisual stimuli, and even individual differences. In order to better understand how temporal integration of audiovisual events occurs in the real world, we need to close the gap between the experimental setting and the complex setting of everyday life. With this work, we aimed to contribute one brick to the bridge that will close this gap. We compared perceived synchrony for long-running and eventful audiovisual sequences to shorter sequences that contain a single audiovisual event, for three types of content: action, music, and speech. The resulting windows of temporal integration showed that participants were better at detecting asynchrony for the longer stimuli, possibly because the long-running sequences contain multiple corresponding events that offer audiovisual timing cues. Moreover, the points of subjective simultaneity differ between content types, suggesting that the nature of a visual scene could influence the temporal perception of events. An expected outcome from this type of experiment was the rich variation among participants{\textquoteright} distributions and the derived points of subjective simultaneity. Hence, the designs of similar experiments call for more participants than traditional psychophysical studies. Heeding this caution, we conclude that existing theories on multisensory perception are ready to be tested on more natural and representative stimuli.},
	doi = {10.3389/fpsyg.2015.00736},
	author = {Eg, Ragnhild and Dawn Marie {Behne}}
}
@inproceedings {Simula.simula.2092,
	title = {Temporal integration for live conversational speech},
	journal = {Proceedings of the 12th International Conference on Auditory-Visual Speech Processing (AVSP2013)},
	number = {12},
	year = {2013},
	month = {August},
	pages = {129-133},
	publisher = {AVSP},
	type = {Workshop},
	abstract = {The difficulty in detecting short asynchronies between corresponding audio and video signals demonstrates the remarkable resilience of the perceptual system when integrating the senses. Thresholds for perceived synchrony vary depending on the complexity, congruency and predictability of the audiovisual event. For instance, asynchrony is typically detected sooner for simple flash and tone combinations than for speech stimuli. In applied scenarios, such as teleconference platforms, the thresholds themselves are of particular interest; since the transmission of audio and video streams can result in temporal misalignments, system providers need to establish how much delay they can allow. This study compares the perception of synchrony in speech for a live two-way teleconference scenario and a controlled experimental set-up. Although methodologies and measures differ, our explorative analysis indicates that the windows of temporal integration are similar for the two scenarios. Nevertheless, the direction of temporal tolerance differs; for the teleconference, audio lead asynchrony was more difficult to detect than for the experimental speech videos. While the windows of temporal integration are fairly independent of the context, the skew in the audio lead threshold may be a reflection of the natural diversion of attending to a conversation.},
	keywords = {Workshop},
	author = {Eg, Ragnhild and Dawn Marie {Behne}},
	editor = {Ouni, Slim and Berthommier, Fr{\'e}d{\'e}ric and Jesse, Alexandra}
}
@misc {Simula.simula.1718,
	title = {Short and sweet, or long and complex? Perceiving temporal synchrony in audiovisual events},
	year = {2012},
	month = {June},
	abstract = {Perceived synchrony varies depending on the audiovisual event. Typically, asynchrony is tolerated at greater lead- and lag-times for speech and music than for action events. The tolerance for asynchrony in speech has been attributed to the unity assumption, which proposes a bonding of auditory and visual speech cues through associations in several dimensions. However, the variations in synchrony perception for different audiovisual events may simply be related to their complexity; where speech and music fluctuate naturally, actions involve isolated events and anticipated moments of impact. The current study measured perception of synchrony for long (13 seconds) and short (1 second) variants of three types of stimuli: (1) action, represented by a game of chess, (2) music, played by a drummer, and (3) speech, presented by an anchorwoman in a newscast. The long variants allowed events to play out with their natural dynamics, whereas short variants offered controlled and predictable single actions or events, selected from the longer segments. Results show that among the long stimuli, lead asynchrony was detected sooner for speech than for chess. This contrasts both with previous research and our own predictions, although it may be related to characteristics of the selected chess scene. Interestingly, tolerance to asynchrony was generally greater for short, than for long, stimuli, especially for speech. These findings suggest that the dynamics of complex events cannot account for previously observed differences in synchrony perception between speech and action events.},
	keywords = {Workshop},
	doi = {10.1163/187847612X647298},
	author = {Eg, Ragnhild and Dawn Marie {Behne}}
}
@article {Simula.mpg.12,
	title = {Audio-Visual Identification of Place of Articulation and Voicing in White and Babble Noise},
	journal = {Journal of the Acoustical Society of America},
	volume = {126},
	number = {1},
	year = {2009},
	month = {July},
	pages = {377-387},
	publisher = {Acoustical Society of America},
	abstract = {Research shows that noise and phonetic attributes influence the degree to which auditory and visual modalities are used in audio-visual speech perception (AVSP). Research has, however, mainly focused on white noise and single phonetic attributes, thus neglecting the more common babble noise and possible interactions between phonetic attributes. This study explores whether white and babble noise differentially influence AVSP and whether these differences depend on phonetic attributes. White and babble noise of 0 and {-}12 dB signal-to-noise ratio were added to congruent and incongruent audio-visual stop consonant-vowel stimuli. The audio (A) and video (V) of incongruent stimuli differed either in place of articulation (POA) or voicing. Responses from 15 young adults show that, compared to white noise, babble resulted in more audio responses for POA stimuli, and fewer for voicing stimuli. Voiced syllables received more audio responses than voiceless syllables. Results can be attributed to discrepancies in the acoustic spectra of both the noise and speech target. Voiced consonants may be more auditorily salient than voiceless consonants which are more spectrally similar to white noise. Visual cues contribute to identification of voicing, but only if the POA is visually salient and auditorily susceptible to the noise type.},
	doi = {10.1121/1.3129508},
	author = {Alm, Magnus and Dawn Marie {Behne} and Wang, Yue and Eg, Ragnhild}
}
@inproceedings {Simula.mpg.14,
	title = {Distorted Visual Information Influences Audiovisual Perception of Voicing},
	journal = {Interspeech-2009},
	year = {2009},
	month = {September},
	pages = {2903-2906},
	publisher = {International Speech Communication Association},
	type = {Conference},
	abstract = {Research has shown that visual information becomes less reliable when images are severely distorted. Furthermore, while voicing is generally identified from acoustical cues, it may also provide perception with visual cues. The current study investigated the impact of video distortion on the audiovisual perception of voicing. Audiovisual stimuli were presented to 30 participants with the original video quality, or with reduced video resolution (75x60 pixels, 45x36 pixels). Results revealed that in addition to increased auditory reliance with video distortion, particularly for voiceless stimuli, perception of voiceless stimuli was more influenced by the visual modality than voiced stimuli.},
	doi = {10.13140/2.1.3302.7525},
	author = {Eg, Ragnhild and Dawn Marie {Behne}},
	editor = {Uther, Maria and Moore, Roger and Cox, Stephen}
}
