positional stream audio

support/client/lib/vwf/model/aframe.js

@@ -554,6 +554,29 @@ define(["module", "vwf/model", "vwf/utility"], function (module, model, utility)
                     }
                 }
 
+                if (value === undefined && aframeObject.nodeName == "A-SOUND") {
+                    value = propertyValue;
+
+                    switch (propertyName) {
+
+                        case "src":
+                            aframeObject.setAttribute('src', propertyValue);
+                            break;
+
+                        case "on":
+                            aframeObject.setAttribute('on', propertyValue);
+                            break;
+
+                        case "autoplay":
+                            aframeObject.setAttribute('autoplay', propertyValue);
+                            break;
+
+                        default:
+                            value = undefined;
+                            break;
+                    }
+                }
+
                 if (value === undefined && aframeObject.nodeName == "A-PLANE") {
                     value = propertyValue;
 
@@ -1006,6 +1029,21 @@ define(["module", "vwf/model", "vwf/utility"], function (module, model, utility)
                     }
                 }
 
+                if (value === undefined && aframeObject.nodeName == "A-SOUND") {
+
+                    switch (propertyName) {
+                        case "src":
+                            value = aframeObject.getAttribute('src');
+                            break;
+                        case "on":
+                            value = aframeObject.getAttribute('on');
+                            break;
+                        case "autoplay":
+                            value = aframeObject.getAttribute('autoplay');
+                            break;
+                    }
+                }
+
                 if (value === undefined && aframeObject.nodeName == "A-GLTF-MODEL") {
                     
                                         switch (propertyName) {
@@ -1061,6 +1099,8 @@ define(["module", "vwf/model", "vwf/utility"], function (module, model, utility)
             aframeObj = document.createElement('a-collada-model');
         } else if (self.state.isAFrameClass(protos, "http://vwf.example.com/aframe/aobjmodel.vwf")) {
             aframeObj = document.createElement('a-obj-model');
+        } else if (self.state.isAFrameClass(protos, "http://vwf.example.com/aframe/asound.vwf")) {
+            aframeObj = document.createElement('a-sound');
          } else if (self.state.isAFrameClass(protos, "http://vwf.example.com/aframe/agltfmodel.vwf")) {
             aframeObj = document.createElement('a-gltf-model');
         } else if (self.state.isAFrameClass(protos, "http://vwf.example.com/aframe/asphere.vwf")) {

support/client/lib/vwf/model/aframe/addon/aframe-components.js

@@ -418,4 +418,100 @@ AFRAME.registerComponent('gearvrcontrol', {
         
             tick: function (t) {
             }
-        })
+        })
+
+        AFRAME.registerComponent('streamsound', {
+
+            schema: {
+                positional: { default: true }
+              },
+           
+            init: function () {
+                var self = this;
+
+                let driver = vwf.views["vwf/view/webrtc"];
+
+                this.listener = null;
+                this.stream = null;
+
+                if(!this.sound) {
+                    this.setupSound();
+                  }
+
+                  if (driver) {
+                    //let avatarID = 'avatar-' + vwf.moniker();
+                    let avatarID = this.el.id.slice(0, 27); //avatar-0RtnYBBTBU84OCNcAAFY
+                   let client = driver.state.clients[avatarID];
+                   if (client ){
+                       if (client.connection) {
+                    this.stream = client.connection.stream;
+                    if (this.stream){
+                        this.audioEl = new Audio();
+                        this.audioEl.srcObject = this.stream;
+            
+                    this.sound.setNodeSource(this.sound.context.createMediaStreamSource(this.stream));
+                    }
+                   }
+                }
+                  }
+                 
+
+                // console.log(this.el);
+
+                // let assets = document.querySelector('a-assets');
+                // let driver = vwf.views["vwf/view/webrtc"];
+                // if (driver) { 
+                //     let stream = driver.local.stream;
+                //     console.log(driver.local.stream);
+                //     console.log(this.el.sceneEl.audioListener)
+                //     let sound = this.el.components.sound;
+                //     console.log(self.sound);
+                   // let audioContext = sound.listener.context;
+                    //var source = audioContext.createMediaStreamSource(stream);
+                
+               // var audioCtx = new AudioContext();
+              //  var source = audioCtx.createMediaStreamSource(stream);
+
+              //  }
+
+            },
+
+            setupSound: function() {
+                var el = this.el;
+                var sceneEl = el.sceneEl;
+            
+                if (this.sound) {
+                  el.removeObject3D(this.attrName);
+                }
+            
+                if (!sceneEl.audioListener) {
+                  sceneEl.audioListener = new THREE.AudioListener();
+                  sceneEl.camera && sceneEl.camera.add(sceneEl.audioListener);
+                  sceneEl.addEventListener('camera-set-active', function(evt) {
+                    evt.detail.cameraEl.getObject3D('camera').add(sceneEl.audioListener);
+                  });
+                }
+                this.listener = sceneEl.audioListener;
+            
+                this.sound = this.data.positional
+                  ? new THREE.PositionalAudio(this.listener)
+                  : new THREE.Audio(this.listener);
+                el.setObject3D(this.attrName, this.sound);
+              },
+
+              remove: function() {
+                if (!this.sound) return;
+            
+                this.el.removeObject3D(this.attrName);
+                if (this.stream) {
+                  this.sound.disconnect();
+                }
+              },
+
+            update: function (old) {
+
+            },
+
+            tick: function (t) {
+            }
+            })

support/client/lib/vwf/model/aframeComponent.js

@@ -808,6 +808,15 @@ define(["module", "vwf/model", "vwf/utility"], function (module, model, utility)
 
         }
 
+        if (self.state.isComponentClass(protos, "http://vwf.example.com/aframe/streamSoundComponent.vwf")) {
+            
+            
+            // aframeObj.el.setAttribute(node.type, {});
+            aframeObj.compName = "streamsound";
+            aframeObj.el.setAttribute(aframeObj.compName, {});
+
+        }
+
         if (self.state.isComponentClass(protos, "http://vwf.example.com/aframe/linepath.vwf")) {
             
             

support/client/lib/vwf/view/webrtc.js

@@ -138,6 +138,7 @@ define( [ "module", "vwf/view", "vwf/utility", "vwf/utility/color", "jquery" ],
                                 peer.connection && peer.connection.disconnect();
                                 let peername = 'avatar-' + peerMoniker;
                                 deletePeerConnection.call( this, peername);
+                               
                             }
                         }
     
@@ -181,7 +182,7 @@ define( [ "module", "vwf/view", "vwf/utility", "vwf/utility/color", "jquery" ],
                 if (micui) micuicomp.on = false;
 
                 this.deleteConnection(nodeID);
-
+                this.kernel.callMethod(nodeID, "removeSoundWebRTC");
             }
 
         },
@@ -436,6 +437,14 @@ define( [ "module", "vwf/view", "vwf/utility", "vwf/utility/color", "jquery" ],
 
           if (local) video.setAttribute("muted", true);
         
+          let audioID = '#audio-' + id;
+          var audio = document.querySelector(audioID);
+          if (!audio) {
+            audio = document.createElement('audio');
+          }
+          audio.setAttribute('id', audioID);
+          //<audio id="river" src="river.mp3" preload="auto"></audio>
+
           var assets = document.querySelector('a-assets');
         
         //   if (!assets) {
@@ -447,7 +456,11 @@ define( [ "module", "vwf/view", "vwf/utility", "vwf/utility/color", "jquery" ],
             assets.appendChild(video);
           }
         
-          return video;
+          if (!assets.contains(audio)) {
+            assets.appendChild(audio);
+          }
+
+          return {'video': video, 'audio': audio};
         }
 
 
@@ -481,11 +494,18 @@ define( [ "module", "vwf/view", "vwf/utility", "vwf/utility/color", "jquery" ],
 
     function displayVideo( id, stream, url, name, destMoniker, local) {
         
-        let video = createVideoElementAsAsset(name, local);
+        let va = createVideoElementAsAsset(name, local);
         //video.setAttribute('src', url);
-        video.srcObject = stream;
+        va.video.srcObject = stream;
+
+        //var audioCtx = new AudioContext();
+        //var source = audioCtx.createMediaStreamSource(stream);
+        //va.audio.src = stream;
 
+        let audioID = 'audio-' + name;
         this.kernel.callMethod( 'avatar-'+id, "setVideoTexture", [name]);
+        this.kernel.callMethod( 'avatar-'+id, "setSoundWebRTC", [audioID]);
+        
 
         return id;
     }
@@ -505,7 +525,7 @@ define( [ "module", "vwf/view", "vwf/utility", "vwf/utility/color", "jquery" ],
     }
 
     function displayRemote( id, stream, url, name, destMoniker, color ) {
-        return displayVideo.call( this, id, stream, url, name, destMoniker, false );
+        return displayVideo.call( this, id, stream, url, name, destMoniker, true );
     }
 
     function capture( media ) {

support/proxy/vwf.example.com/aframe/asound.vwf.yaml

@@ -0,0 +1,8 @@
+# https://aframe.io/docs/master/primitives/a-sound.html
+--- 
+extends: http://vwf.example.com/aframe/aentity.vwf
+type: "a-sound"
+properties:
+  src:
+  on:
+  autoplay:

support/proxy/vwf.example.com/aframe/avatar.js

@@ -2,7 +2,7 @@ this.simpleBodyDef = {
     "extends": "http://vwf.example.com/aframe/abox.vwf",
     "properties": {
         "color": "white",
-        "position": "0 0.66 0.5",
+        "position": "0 0.66 0.7",
         "height": 1.3,
         "width": 0.65,
         "depth": 0.1,
@@ -13,7 +13,7 @@ this.modelBodyDef = {
     "extends": "http://vwf.example.com/aframe/agltfmodel.vwf",
     "properties": {
         "src": "#avatar",
-        "position": "0 0 0.7",
+        "position": "0 0 0.8",
         "rotation": "0 180 0"
     },
     "children": {
@@ -60,7 +60,7 @@ this.createAvatarBody = function (modelSrc) {
             "myHead": {
                 "extends": "http://vwf.example.com/aframe/aentity.vwf",
                 "properties": {
-                    "position": "0 1.6 0.5",
+                    "position": "0 1.6 0.7",
                     "visible": true
                 },
                 children: {
@@ -88,7 +88,7 @@ this.createAvatarBody = function (modelSrc) {
                         "id": 'camera-' + this.id,
                         "extends": "http://vwf.example.com/aframe/acamera.vwf",
                         "properties": {
-                            "position": "0 0 -0.5",
+                            "position": "0 0 -0.7",
                             "look-controls-enabled": false,
                             "wasd-controls-enabled": false,
                             "user-height": 0
@@ -320,6 +320,35 @@ this.setVideoTexture = function(val){
     this.avatarNode.myHead.visual.src = '#'+val;
 }
 
+this.removeSoundWebRTC = function(){
+
+    this.avatarNode.children.delete(this.avatarNode.audio);
+}
+
+this.setSoundWebRTC = function(val){
+    console.log(val);
+    if (this.avatarNode.audio) this.removeSoundWebRTC();
+    
+    var soundNode = {
+        "extends": "http://vwf.example.com/aframe/aentity.vwf",
+        "properties": {
+        },
+        "children":{
+            "streamsound":{
+                "extends": "http://vwf.example.com/aframe/streamSoundComponent.vwf",
+                "type": "component",
+                "properties": {
+                }
+            }
+        }
+    }
+    this.avatarNode.children.create("audio", soundNode );
+   // this.setSmallVideoHead();
+
+    //this.avatarNode.audio.src = '#tempAudio';
+    //this.avatarNode.audio.src = '#'+val;
+}
+
 this.webrtcTurnOnOff = function(val){
     console.log('WEBRTC is ', val);
 }

support/proxy/vwf.example.com/aframe/avatar.vwf.yaml

@@ -40,5 +40,9 @@ methods:
     webrtcMuteVideo:
         parameters:
             - val
+    setSoundWebRTC:
+         parameters:
+            - val
+    removeSoundWebRTC:
 scripts:
 - source: "http://vwf.example.com/aframe/avatar.js"

support/proxy/vwf.example.com/aframe/streamSoundComponent.vwf.yaml

@@ -0,0 +1,5 @@
+# StreamSoundComponent
+---
+extends: http://vwf.example.com/aframe/aentityComponent.vwf
+type: "component"
+properties: